From 4e983a583039844b85fcc7dd5b18f7a49698174c Mon Sep 17 00:00:00 2001 From: Matt Graham Date: Mon, 17 Jun 2024 13:50:06 +0100 Subject: [PATCH 01/19] Ensure individual properties updated across modules in generic first appointments (#1387) * Allow updating memoized population dataframe row view * Renaming patient_id and patient_details * Updates to individual properties via IndividualProperties object * Remove dot based attributed access from IndividualProperties * Remove unused random_state argument * Refactor and rename HSI_BaseGenericFirstAppt * Remove unused target_is_alive property * Move do_at_generic_first_appt outside of core * Pass in schedule_hsi_event to generic first appointment actions * Minor pylint fixes * Exclude GenericFirstApptsModule from enumerations * Subclass Hiv and Epilepsy from GenericFirstApptsModule * Decouple Population class from Simulation to allow easier testing * Fix import order to satisfy isor * Remove unused sim slot from Population * Add unit tests for population and individual properties * Fix errors from bad manual merge * Remove unused schedule_hsi_event property * Make do_at_generic_first_appt methods keyword argument only * Tidy up docstrings * Avoid accidental leaking of methods details into core * Changing from subclass to mix-in * Refactor individual properties context manager * Fix use of context manager in tests * Correct symptoms type hint --- src/tlo/analysis/hsi_events.py | 2 +- src/tlo/core.py | 98 +----- src/tlo/methods/alri.py | 29 +- src/tlo/methods/bladder_cancer.py | 24 +- src/tlo/methods/breast_cancer.py | 20 +- src/tlo/methods/cardio_metabolic_disorders.py | 62 ++-- src/tlo/methods/chronicsyndrome.py | 20 +- src/tlo/methods/copd.py | 52 ++-- src/tlo/methods/demography.py | 4 +- src/tlo/methods/depression.py | 66 ++-- src/tlo/methods/diarrhoea.py | 22 +- src/tlo/methods/epilepsy.py | 20 +- src/tlo/methods/healthseekingbehaviour.py | 20 +- src/tlo/methods/hiv.py | 19 +- src/tlo/methods/hsi_event.py | 7 - src/tlo/methods/hsi_generic_first_appts.py | 287 ++++++++++++------ src/tlo/methods/labour.py | 30 +- src/tlo/methods/malaria.py | 65 ++-- src/tlo/methods/measles.py | 20 +- src/tlo/methods/mockitis.py | 20 +- src/tlo/methods/oesophagealcancer.py | 20 +- src/tlo/methods/other_adult_cancers.py | 20 +- src/tlo/methods/pregnancy_supervisor.py | 26 +- src/tlo/methods/prostate_cancer.py | 24 +- src/tlo/methods/rti.py | 70 +++-- src/tlo/methods/schisto.py | 20 +- src/tlo/methods/stunting.py | 48 +-- src/tlo/population.py | 153 +++++++--- src/tlo/simulation.py | 7 +- src/tlo/test/random_birth.py | 2 +- tests/test_basic_sims.py | 2 +- tests/test_diarrhoea.py | 59 ++-- tests/test_malaria.py | 4 +- tests/test_module_dependencies.py | 6 +- tests/test_population.py | 216 +++++++++++++ tests/test_stunting.py | 44 ++- 36 files changed, 994 insertions(+), 614 deletions(-) create mode 100644 tests/test_population.py diff --git a/src/tlo/analysis/hsi_events.py b/src/tlo/analysis/hsi_events.py index 1a9d889ce4..9bc973f67c 100644 --- a/src/tlo/analysis/hsi_events.py +++ b/src/tlo/analysis/hsi_events.py @@ -47,7 +47,7 @@ def get_hsi_event_classes_per_module( module = importlib.import_module(f'tlo.methods.{module_name}') tlo_module_classes = [ obj for _, obj in inspect.getmembers(module) - if is_valid_tlo_module_subclass(obj, excluded_modules) + if is_valid_tlo_module_subclass(obj, {}) ] hsi_event_classes = [ obj for _, obj in inspect.getmembers(module) diff --git a/src/tlo/core.py b/src/tlo/core.py index 98553c4039..fe92203e56 100644 --- a/src/tlo/core.py +++ b/src/tlo/core.py @@ -8,28 +8,16 @@ import json from enum import Enum, auto -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, TypeAlias, Union +from typing import TYPE_CHECKING import numpy as np import pandas as pd if TYPE_CHECKING: - from numpy.random import RandomState + from typing import Optional - from tlo.methods.healthsystem import HealthSystem - from tlo.population import PatientDetails from tlo.simulation import Simulation -DiagnosisFunction: TypeAlias = Callable[[str, bool, bool], Any] -ConsumablesChecker: TypeAlias = Callable[ - [ - Union[None, np.integer, int, List, Set, Dict], - Union[None, np.integer, int, List, Set, Dict], - ], - Union[bool, Dict], -] -IndividualPropertyUpdates: TypeAlias = Dict[str, Any] - class Types(Enum): """Possible types for parameters and properties. @@ -253,9 +241,6 @@ class attribute on a subclass. # parameters created from the PARAMETERS specification. __slots__ = ('name', 'parameters', 'rng', 'sim') - @property - def healthsystem(self) -> HealthSystem: - return self.sim.modules["HealthSystem"] def __init__(self, name=None): """Construct a new disease module ready to be included in a simulation. @@ -391,82 +376,3 @@ def on_birth(self, mother_id, child_id): def on_simulation_end(self): """This is called after the simulation has ended. Modules do not need to declare this.""" - - def do_at_generic_first_appt( - self, - patient_id: int, - patient_details: Optional[PatientDetails], - symptoms: Optional[List[str]], - diagnosis_function: Optional[DiagnosisFunction], - consumables_checker: Optional[ConsumablesChecker], - facility_level: Optional[str], - treatment_id: Optional[str], - random_state: Optional[RandomState], - ) -> Union[IndividualPropertyUpdates, None]: - """ - Actions to be take during a NON-emergency generic HSI. - - Derived classes should overwrite this method so that they are - compatible with the HealthSystem module, and can schedule HSI - events when a patient presents symptoms indicative of the - corresponding illness or condition. - - When overwriting, arguments that are not required can be left out - of the definition. - If done so, the method MUST take a **kwargs input to avoid errors - when looping over all disease modules and running their generic - HSI methods. - - HSI_Events should be scheduled by the Module implementing this - method using the :py:meth:`Module.healthsystem.schedule_hsi` method. - However, they should not write updates back to the population - DataFrame in this method - these values should be returned as a - dictionary as described below: - - The return value of this function should be a dictionary - containing any changes that need to be made to the individual's - row in the population DataFrame. - Key/value pairs should be the column name and the new value to - assign to the patient. - In the event no updates are required; return an object that evaluates - to False when cast to a bool. Your options are: - - Omit a return statement and value (preferred). - - Return an empty dictionary. Use this case when patient details - might need updating conditionally, on EG patient symptoms or consumable - availability. In which case, an empty dictionary should be created and - key-value pairs added to this dictionary as such conditionals are checked. - If no conditionals are met, the empty dictionary will be returned. - - Use a return statement with no values (use if the logic of your - module-specific method necessitates the explicit return). - - Return None (not recommended, use "return" on its own, as above). - - :param patient_id: Row index (ID) of the individual target of the HSI event in the population DataFrame. - :param patient_details: Patient details as provided in the population DataFrame. - :param symptoms: List of symptoms the patient is experiencing. - :param diagnosis_function: A function that can run diagnosis tests based on the patient's symptoms. - :param consumables_checker: A function that can query the HealthSystem to check for available consumables. - :param facility_level: The level of the facility that the patient presented at. - :param treatment_id: The treatment id of the HSI event triggering the generic appointment. - :param random_state: Random number generator to be used when making random choices during event creation. - """ - - def do_at_generic_first_appt_emergency( - self, - patient_id: int, - patient_details: Optional[PatientDetails] = None, - symptoms: Optional[List[str]] = None, - diagnosis_function: Optional[DiagnosisFunction] = None, - consumables_checker: Optional[ConsumablesChecker] = None, - facility_level: Optional[str] = None, - treatment_id: Optional[str] = None, - random_state: Optional[RandomState] = None, - ) -> Union[IndividualPropertyUpdates, None]: - """ - Actions to be take during an EMERGENCY generic HSI. - Call signature and return values are identical to the - :py:meth:`~Module.do_at_generic_first_appt` method. - Derived classes should overwrite this method so that they are - compatible with the HealthSystem module, and can schedule HSI - events when a patient presents symptoms indicative of the - corresponding illness or condition. - """ diff --git a/src/tlo/methods/alri.py b/src/tlo/methods/alri.py index 61e9ae848b..8c1ab41401 100644 --- a/src/tlo/methods/alri.py +++ b/src/tlo/methods/alri.py @@ -30,17 +30,18 @@ import pandas as pd from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata from tlo.methods.causes import Cause from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom from tlo.util import random_date, sample_outcome if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + from tlo.population import IndividualProperties logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -54,7 +55,7 @@ # --------------------------------------------------------------------------------------------------------- -class Alri(Module): +class Alri(Module, GenericFirstAppointmentsMixin): """This is the disease module for Acute Lower Respiratory Infections.""" INIT_DEPENDENCIES = { @@ -1362,39 +1363,35 @@ def _ultimate_treatment_indicated_for_patient(classification_for_treatment_decis def do_at_generic_first_appt( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, facility_level: str, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: # Action taken when a child (under 5 years old) presents at a # generic appointment (emergency or non-emergency) with symptoms # of `cough` or `difficult_breathing`. - if patient_details.age_years <= 5 and ( + if individual_properties["age_years"] <= 5 and ( ("cough" in symptoms) or ("difficult_breathing" in symptoms) ): self.record_sought_care_for_alri() # All persons have an initial out-patient appointment at the current facility level. event = HSI_Alri_Treatment( - person_id=patient_id, module=self, facility_level=facility_level + person_id=person_id, module=self, facility_level=facility_level ) - self.healthsystem.schedule_hsi_event( + schedule_hsi_event( event, topen=self.sim.date, tclose=self.sim.date + pd.DateOffset(days=1), priority=1, ) - def do_at_generic_first_appt_emergency( - self, - **kwargs, - ) -> IndividualPropertyUpdates: + def do_at_generic_first_appt_emergency(self, **kwargs) -> None: # Emergency and non-emergency treatment is identical for alri - return self.do_at_generic_first_appt( - **kwargs, - ) + self.do_at_generic_first_appt(**kwargs) class Models: diff --git a/src/tlo/methods/bladder_cancer.py b/src/tlo/methods/bladder_cancer.py index 4c94fd8f51..78899d4705 100644 --- a/src/tlo/methods/bladder_cancer.py +++ b/src/tlo/methods/bladder_cancer.py @@ -13,7 +13,6 @@ import pandas as pd from tlo import DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata @@ -22,16 +21,18 @@ from tlo.methods.demography import InstantaneousDeath from tlo.methods.dxmanager import DxTest from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + from tlo.population import IndividualProperties logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class BladderCancer(Module): +class BladderCancer(Module, GenericFirstAppointmentsMixin): """Bladder Cancer Disease Module""" def __init__(self, name=None, resourcefilepath=None): @@ -595,27 +596,28 @@ def report_daly_values(self): def do_at_generic_first_appt( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: # Only investigate if the patient is not a child - if patient_details.age_years > 5: + if individual_properties["age_years"] > 5: # Begin investigation if symptoms are present. if "blood_urine" in symptoms: event = HSI_BladderCancer_Investigation_Following_Blood_Urine( - person_id=patient_id, module=self + person_id=person_id, module=self ) - self.healthsystem.schedule_hsi_event( + schedule_hsi_event( event, topen=self.sim.date, priority=0 ) if "pelvic_pain" in symptoms: event = HSI_BladderCancer_Investigation_Following_pelvic_pain( - person_id=patient_id, module=self + person_id=person_id, module=self ) - self.healthsystem.schedule_hsi_event( + schedule_hsi_event( event, topen=self.sim.date, priority=0 ) diff --git a/src/tlo/methods/breast_cancer.py b/src/tlo/methods/breast_cancer.py index 21347c1f98..9ef5dc3e41 100644 --- a/src/tlo/methods/breast_cancer.py +++ b/src/tlo/methods/breast_cancer.py @@ -12,7 +12,6 @@ import pandas as pd from tlo import DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata @@ -21,16 +20,18 @@ from tlo.methods.demography import InstantaneousDeath from tlo.methods.dxmanager import DxTest from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + from tlo.population import IndividualProperties logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class BreastCancer(Module): +class BreastCancer(Module, GenericFirstAppointmentsMixin): """Breast Cancer Disease Module""" def __init__(self, name=None, resourcefilepath=None): @@ -572,19 +573,20 @@ def report_daly_values(self): def do_at_generic_first_appt( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: # If the patient is not a child and symptoms include breast # lump discernible - if patient_details.age_years > 5 and "breast_lump_discernible" in symptoms: + if individual_properties["age_years"] > 5 and "breast_lump_discernible" in symptoms: event = HSI_BreastCancer_Investigation_Following_breast_lump_discernible( - person_id=patient_id, + person_id=person_id, module=self, ) - self.healthsystem.schedule_hsi_event(event, topen=self.sim.date, priority=0) + schedule_hsi_event(event, topen=self.sim.date, priority=0) # --------------------------------------------------------------------------------------------------------- diff --git a/src/tlo/methods/cardio_metabolic_disorders.py b/src/tlo/methods/cardio_metabolic_disorders.py index 1d5f47ecb9..c46ea7b37e 100644 --- a/src/tlo/methods/cardio_metabolic_disorders.py +++ b/src/tlo/methods/cardio_metabolic_disorders.py @@ -21,7 +21,6 @@ import pandas as pd from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata @@ -29,11 +28,13 @@ from tlo.methods.causes import Cause from tlo.methods.dxmanager import DxTest from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom from tlo.util import random_date if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + from tlo.population import IndividualProperties # --------------------------------------------------------------------------------------------------------- # MODULE DEFINITIONS @@ -42,7 +43,7 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class CardioMetabolicDisorders(Module): +class CardioMetabolicDisorders(Module, GenericFirstAppointmentsMixin): """ CardioMetabolicDisorders module covers a subset of cardio-metabolic conditions and events. Conditions are binary and individuals experience a risk of acquiring or losing a condition based on annual probability and @@ -814,19 +815,20 @@ def on_hsi_alert(self, person_id, treatment_id): def do_at_generic_first_appt( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs - ) -> IndividualPropertyUpdates: + ) -> None: # This is called by the HSI generic first appts module whenever a # person attends an appointment and determines if the person will # be tested for one or more conditions. # A maximum of one instance of `HSI_CardioMetabolicDisorders_Investigations` # is created for the person, during which multiple conditions can # be investigated. - if patient_details.age_years <= 5: - return {} + if individual_properties["age_years"] <= 5: + return # The list of conditions that will be investigated in follow-up HSI conditions_to_investigate = [] @@ -835,11 +837,11 @@ def do_at_generic_first_appt( # Determine if there are any conditions that should be investigated: for condition in self.conditions: - is_already_diagnosed = getattr(patient_details, f"nc_{condition}_ever_diagnosed") + is_already_diagnosed = individual_properties[ + f"nc_{condition}_ever_diagnosed" + ] has_symptom = f"{condition}_symptoms" in symptoms - date_of_last_test = getattr( - patient_details, f"nc_{condition}_date_last_test" - ) + date_of_last_test = individual_properties[f"nc_{condition}_date_last_test"] next_test_due = ( pd.isnull(date_of_last_test) or (self.sim.date - date_of_last_test).days > DAYS_IN_YEAR / 2 @@ -867,34 +869,32 @@ def do_at_generic_first_appt( if conditions_to_investigate: event = HSI_CardioMetabolicDisorders_Investigations( module=self, - person_id=patient_id, + person_id=person_id, conditions_to_investigate=conditions_to_investigate, has_any_cmd_symptom=has_any_cmd_symptom, ) - self.healthsystem.schedule_hsi_event(event, topen=self.sim.date, priority=0) + schedule_hsi_event(event, topen=self.sim.date, priority=0) def do_at_generic_first_appt_emergency( self, - patient_id: int, - patient_details: PatientDetails = None, - symptoms: List[str] = None, + person_id: int, + individual_properties: IndividualProperties, + symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs, - ) -> IndividualPropertyUpdates: - # This is called by the HSI generic first appts module whenever - # a person attends an emergency appointment and determines if they - # will receive emergency care based on the duration of time since - # symptoms have appeared. A maximum of one instance of - # `HSI_CardioMetabolicDisorders_SeeksEmergencyCareAndGetsTreatment` - # is created for the person, during which multiple events can be - # investigated. + ) -> None: + # This is called by the HSI generic first appts module whenever a person attends + # an emergency appointment and determines if they will receive emergency care + # based on the duration of time since symptoms have appeared. A maximum of one + # instance of `HSI_CardioMetabolicDisorders_SeeksEmergencyCareAndGetsTreatment` + # is created for the person, during which multiple events can be investigated. ev_to_investigate = [] for ev in self.events: - # If the person has symptoms of damage from within the last 3 days, - # schedule them for emergency care + # If the person has symptoms of damage from within the last 3 days, schedule + # them for emergency care if f"{ev}_damage" in symptoms and ( ( - self.sim.date - - getattr(patient_details, f"nc_{ev}_date_last_event") + self.sim.date - individual_properties[f"nc_{ev}_date_last_event"] ).days <= 3 ): @@ -903,10 +903,10 @@ def do_at_generic_first_appt_emergency( if ev_to_investigate: event = HSI_CardioMetabolicDisorders_SeeksEmergencyCareAndGetsTreatment( module=self, - person_id=patient_id, + person_id=person_id, events_to_investigate=ev_to_investigate, ) - self.healthsystem.schedule_hsi_event(event, topen=self.sim.date, priority=1) + schedule_hsi_event(event, topen=self.sim.date, priority=1) class Tracker: diff --git a/src/tlo/methods/chronicsyndrome.py b/src/tlo/methods/chronicsyndrome.py index 8d466149d8..0ae6599939 100644 --- a/src/tlo/methods/chronicsyndrome.py +++ b/src/tlo/methods/chronicsyndrome.py @@ -1,22 +1,27 @@ -from typing import List +from __future__ import annotations + +from typing import TYPE_CHECKING, List import numpy as np import pandas as pd from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.methods import Metadata from tlo.methods.causes import Cause from tlo.methods.demography import InstantaneousDeath from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom +if TYPE_CHECKING: + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class ChronicSyndrome(Module): +class ChronicSyndrome(Module, GenericFirstAppointmentsMixin): """ This is a dummy chronic disease It demonstrates the following behaviours in respect of the healthsystem module: @@ -280,17 +285,18 @@ def report_daly_values(self): def do_at_generic_first_appt_emergency( self, - patient_id: int, + person_id: int, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: """Example for CHRONIC SYNDROME""" if "craving_sandwiches" in symptoms: event = HSI_ChronicSyndrome_SeeksEmergencyCareAndGetsTreatment( module=self, - person_id=patient_id, + person_id=person_id, ) - self.healthsystem.schedule_hsi_event(event, topen=self.sim.date, priority=1) + schedule_hsi_event(event, topen=self.sim.date, priority=1) class ChronicSyndromeEvent(RegularEvent, PopulationScopeEventMixin): diff --git a/src/tlo/methods/copd.py b/src/tlo/methods/copd.py index a39e263280..dc85f2b5d9 100644 --- a/src/tlo/methods/copd.py +++ b/src/tlo/methods/copd.py @@ -7,17 +7,18 @@ from tlo import Module, Parameter, Property, Types, logging from tlo.analysis.utils import flatten_multi_index_series_into_dict_for_logging -from tlo.core import ConsumablesChecker, IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata from tlo.methods.causes import Cause from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom from tlo.util import random_date if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import ConsumablesChecker, HSIEventScheduler + from tlo.population import IndividualProperties logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -29,7 +30,7 @@ } -class Copd(Module): +class Copd(Module, GenericFirstAppointmentsMixin): """The module responsible for determining Chronic Obstructive Pulmonary Diseases (COPD) status and outcomes. and initialises parameters and properties associated with COPD plus functions and events related to COPD.""" @@ -212,9 +213,10 @@ def do_logging(self): def _common_first_appt( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, consumables_checker: ConsumablesChecker, ): """What to do when a person presents at the generic first appt HSI @@ -223,51 +225,53 @@ def _common_first_appt( * Otherwise --> just give inhaler. """ if ('breathless_moderate' in symptoms) or ('breathless_severe' in symptoms): - patient_details_updates = {} # Give inhaler if patient does not already have one - if not patient_details.ch_has_inhaler: - if consumables_checker({self.item_codes["bronchodilater_inhaler"]: 1}): - patient_details_updates["ch_has_inhaler"] = True - + if not individual_properties["ch_has_inhaler"] and consumables_checker( + {self.item_codes["bronchodilater_inhaler"]: 1} + ): + individual_properties["ch_has_inhaler"] = True if "breathless_severe" in symptoms: event = HSI_Copd_TreatmentOnSevereExacerbation( - module=self, person_id=patient_id + module=self, person_id=person_id ) - self.healthsystem.schedule_hsi_event( + schedule_hsi_event( event, topen=self.sim.date, priority=0 ) - return patient_details_updates def do_at_generic_first_appt( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, consumables_checker: ConsumablesChecker, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: # Non-emergency appointments are only forwarded if # the patient is over 5 years old - if patient_details.age_years > 5: + if individual_properties["age_years"] > 5: return self._common_first_appt( - patient_id=patient_id, - patient_details=patient_details, + person_id=person_id, + individual_properties=individual_properties, symptoms=symptoms, + schedule_hsi_event=schedule_hsi_event, consumables_checker=consumables_checker, ) def do_at_generic_first_appt_emergency( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, consumables_checker: ConsumablesChecker, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: return self._common_first_appt( - patient_id=patient_id, - patient_details=patient_details, + person_id=person_id, + individual_properties=individual_properties, symptoms=symptoms, + schedule_hsi_event=schedule_hsi_event, consumables_checker=consumables_checker, ) diff --git a/src/tlo/methods/demography.py b/src/tlo/methods/demography.py index f3d3e3a8c4..8d510f29ae 100644 --- a/src/tlo/methods/demography.py +++ b/src/tlo/methods/demography.py @@ -634,11 +634,11 @@ def apply(self, population): df = population.props dates_of_birth = df.loc[df.is_alive, 'date_of_birth'] df.loc[df.is_alive, 'age_exact_years'] = age_at_date( - population.sim.date, dates_of_birth + self.module.sim.date, dates_of_birth ) df.loc[df.is_alive, 'age_years'] = df.loc[df.is_alive, 'age_exact_years'].astype('int64') df.loc[df.is_alive, 'age_range'] = df.loc[df.is_alive, 'age_years'].map(self.age_range_lookup) - df.loc[df.is_alive, 'age_days'] = (population.sim.date - dates_of_birth).dt.days + df.loc[df.is_alive, 'age_days'] = (self.module.sim.date - dates_of_birth).dt.days class OtherDeathPoll(RegularEvent, PopulationScopeEventMixin): diff --git a/src/tlo/methods/depression.py b/src/tlo/methods/depression.py index ef1e4f8cc7..81ae29403e 100644 --- a/src/tlo/methods/depression.py +++ b/src/tlo/methods/depression.py @@ -4,23 +4,24 @@ from __future__ import annotations from pathlib import Path -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, List, Optional, Union import numpy as np import pandas as pd from tlo import DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import DiagnosisFunction, IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata from tlo.methods.causes import Cause from tlo.methods.dxmanager import DxTest from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import DiagnosisFunction, HSIEventScheduler + from tlo.population import IndividualProperties logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -30,7 +31,7 @@ # MODULE DEFINITIONS # --------------------------------------------------------------------------------------------------------- -class Depression(Module): +class Depression(Module, GenericFirstAppointmentsMixin): def __init__(self, name=None, resourcefilepath=None): super().__init__(name) self.resourcefilepath = resourcefilepath @@ -596,20 +597,26 @@ def do_on_presentation_to_care(self, person_id: int, hsi_event: HSI_Event): hsi_event.TREATMENT_ID, self.sim.population.props.at[person_id, "de_ever_diagnosed_depression"], ): - patient_details_updates = self.do_when_suspected_depression( - person_id=person_id, hsi_event=hsi_event + individual_properties = {} + self.do_when_suspected_depression( + person_id=person_id, + individual_properties=individual_properties, + schedule_hsi_event=self.sim.modules["HealthSystem"].schedule_hsi_event, + hsi_event=hsi_event ) - self.sim.population.props.loc[person_id, patient_details_updates.keys()] = ( - patient_details_updates.values() + self.sim.population.props.loc[person_id, individual_properties.keys()] = ( + individual_properties.values() ) return def do_when_suspected_depression( self, person_id: int, + individual_properties: Union[dict, IndividualProperties], + schedule_hsi_event: HSIEventScheduler, diagnosis_function: Optional[DiagnosisFunction] = None, hsi_event: Optional[HSI_Event] = None, - ) -> IndividualPropertyUpdates: + ) -> None: """ This is called by any HSI event when depression is suspected or otherwise investigated. @@ -621,12 +628,11 @@ def do_when_suspected_depression( runs diagnosis tests. :param person_id: Patient's row index in the population DataFrame. + :param individual_properties: Indexable object to write individual property updates to. + :param schedule_hsi_event: Function to schedule subsequent HSI events. :param diagnosis_function: A function capable of running diagnosis checks on the population. :param hsi_event: The HSI_Event that triggered this call. - :returns: Values as per the output of do_at_generic_first_appt(). """ - patient_details_updates = {} - if diagnosis_function is None: assert isinstance( hsi_event, HSI_Event @@ -643,49 +649,51 @@ def diagnosis_function(tests, use_dict: bool = False, report_tried: bool = False # Assess for depression and initiate treatments for depression if positive diagnosis if diagnosis_function('assess_depression'): # If depressed: diagnose the person with depression - patient_details_updates['de_ever_diagnosed_depression'] = True + individual_properties['de_ever_diagnosed_depression'] = True scheduling_options = {"priority": 0, "topen": self.sim.date} # Provide talking therapy # (this can occur even if the person has already had talking therapy before) - self.healthsystem.schedule_hsi_event( + schedule_hsi_event( HSI_Depression_TalkingTherapy(module=self, person_id=person_id), **scheduling_options, ) # Initiate person on anti-depressants # (at the same facility level as the HSI event that is calling) - self.healthsystem.schedule_hsi_event( + schedule_hsi_event( HSI_Depression_Start_Antidepressant(module=self, person_id=person_id), **scheduling_options, ) - return patient_details_updates def do_at_generic_first_appt( - self, - patient_details: PatientDetails, - **kwargs - ) -> IndividualPropertyUpdates: - if patient_details.age_years > 5: - return self.do_at_generic_first_appt_emergency( - patient_details=patient_details, + self, individual_properties: IndividualProperties, **kwargs + ) -> None: + if individual_properties["age_years"] > 5: + self.do_at_generic_first_appt_emergency( + individual_properties=individual_properties, **kwargs, ) def do_at_generic_first_appt_emergency( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, diagnosis_function: DiagnosisFunction, treatment_id: str, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: if self._check_for_suspected_depression( - symptoms, treatment_id, patient_details.de_ever_diagnosed_depression + symptoms, + treatment_id, + individual_properties["de_ever_diagnosed_depression"], ): - return self.do_when_suspected_depression( - person_id=patient_id, + self.do_when_suspected_depression( + person_id=person_id, + individual_properties=individual_properties, diagnosis_function=diagnosis_function, + schedule_hsi_event=schedule_hsi_event, ) diff --git a/src/tlo/methods/diarrhoea.py b/src/tlo/methods/diarrhoea.py index cc589475d5..8ca36ebedb 100644 --- a/src/tlo/methods/diarrhoea.py +++ b/src/tlo/methods/diarrhoea.py @@ -26,17 +26,18 @@ import pandas as pd from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import DiagnosisFunction, IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata from tlo.methods.causes import Cause from tlo.methods.dxmanager import DxTest from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.util import random_date, sample_outcome if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import DiagnosisFunction, HSIEventScheduler + from tlo.population import IndividualProperties logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -46,7 +47,7 @@ # MODULE DEFINITIONS # --------------------------------------------------------------------------------------------------------- -class Diarrhoea(Module): +class Diarrhoea(Module, GenericFirstAppointmentsMixin): # Declare the pathogens that this module will simulate: pathogens = [ 'rotavirus', @@ -948,17 +949,18 @@ def check_properties(self): def do_at_generic_first_appt( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, + schedule_hsi_event: HSIEventScheduler, symptoms: List[str], diagnosis_function: DiagnosisFunction, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: # This routine is called when Diarrhoea is a symptom for a child # attending a Generic HSI Appointment. It checks for danger signs # and schedules HSI Events appropriately. - if patient_details.age_years > 5 or "diarrhoea" not in symptoms: - return {} + if individual_properties["age_years"] > 5 or "diarrhoea" not in symptoms: + return # 1) Assessment of danger signs danger_signs = diagnosis_function( @@ -974,8 +976,8 @@ def do_at_generic_first_appt( HSI_Diarrhoea_Treatment_Inpatient if is_inpatient else HSI_Diarrhoea_Treatment_Outpatient ) - event = hsi_event_class(person_id=patient_id, module=self) - self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date) + event = hsi_event_class(person_id=person_id, module=self) + schedule_hsi_event(event, priority=0, topen=self.sim.date) class Models: diff --git a/src/tlo/methods/epilepsy.py b/src/tlo/methods/epilepsy.py index 8bc71fb69f..a1650a3889 100644 --- a/src/tlo/methods/epilepsy.py +++ b/src/tlo/methods/epilepsy.py @@ -1,23 +1,28 @@ +from __future__ import annotations + from pathlib import Path -from typing import List, Union +from typing import TYPE_CHECKING, List, Union import numpy as np import pandas as pd from tlo import DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.methods import Metadata from tlo.methods.causes import Cause from tlo.methods.demography import InstantaneousDeath from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom +if TYPE_CHECKING: + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class Epilepsy(Module): +class Epilepsy(Module, GenericFirstAppointmentsMixin): def __init__(self, name=None, resourcefilepath=None): super().__init__(name) self.resourcefilepath = resourcefilepath @@ -395,13 +400,14 @@ def get_best_available_medicine(self, hsi_event) -> Union[None, str]: def do_at_generic_first_appt_emergency( self, - patient_id: int, + person_id: int, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: if "seizures" in symptoms: - event = HSI_Epilepsy_Start_Anti_Epileptic(person_id=patient_id, module=self) - self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date) + event = HSI_Epilepsy_Start_Anti_Epileptic(person_id=person_id, module=self) + schedule_hsi_event(event, priority=0, topen=self.sim.date) class EpilepsyEvent(RegularEvent, PopulationScopeEventMixin): diff --git a/src/tlo/methods/healthseekingbehaviour.py b/src/tlo/methods/healthseekingbehaviour.py index f29dc37fcc..22e628d166 100644 --- a/src/tlo/methods/healthseekingbehaviour.py +++ b/src/tlo/methods/healthseekingbehaviour.py @@ -5,23 +5,28 @@ The write-up of these estimates is: Health-seeking behaviour estimates for adults and children.docx """ +from __future__ import annotations + from pathlib import Path -from typing import List +from typing import TYPE_CHECKING, List import numpy as np import pandas as pd from tlo import Date, DateOffset, Module, Parameter, Types -from tlo.core import IndividualPropertyUpdates from tlo.events import PopulationScopeEventMixin, Priority, RegularEvent from tlo.lm import LinearModel from tlo.methods import Metadata from tlo.methods.hsi_generic_first_appts import ( + GenericFirstAppointmentsMixin, HSI_EmergencyCare_SpuriousSymptom, HSI_GenericEmergencyFirstAppt, HSI_GenericNonEmergencyFirstAppt, ) +if TYPE_CHECKING: + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + # --------------------------------------------------------------------------------------------------------- # MODULE DEFINITIONS # --------------------------------------------------------------------------------------------------------- @@ -29,7 +34,7 @@ HIGH_ODDS_RATIO = 1e5 -class HealthSeekingBehaviour(Module): +class HealthSeekingBehaviour(Module, GenericFirstAppointmentsMixin): """ This modules determines if the onset of symptoms will lead to that person presenting at the health facility for a HSI_GenericFirstAppointment. @@ -255,16 +260,17 @@ def force_any_symptom_to_lead_to_healthcareseeking(self): def do_at_generic_first_appt_emergency( self, - patient_id: int, + person_id: int, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: if "spurious_emergency_symptom" in symptoms: event = HSI_EmergencyCare_SpuriousSymptom( module=self.sim.modules["HealthSeekingBehaviour"], - person_id=patient_id, + person_id=person_id, ) - self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date) + schedule_hsi_event(event, priority=0, topen=self.sim.date) # --------------------------------------------------------------------------------------------------------- # REGULAR POLLING EVENT diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py index fde32ed915..591ccc6e3d 100644 --- a/src/tlo/methods/hiv.py +++ b/src/tlo/methods/hiv.py @@ -23,29 +23,33 @@ * Cotrimoxazole is not included - either in effect of consumption of the drug (because the effect is not known). * Calibration has not been done: most things look OK - except HIV-AIDS deaths """ +from __future__ import annotations import os -from typing import List +from typing import TYPE_CHECKING, List import numpy as np import pandas as pd from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata, demography, tb from tlo.methods.causes import Cause from tlo.methods.dxmanager import DxTest from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom from tlo.util import create_age_range_lookup +if TYPE_CHECKING: + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class Hiv(Module): +class Hiv(Module, GenericFirstAppointmentsMixin): """ The HIV Disease Module """ @@ -1568,22 +1572,23 @@ def is_subset(col_for_set, col_for_subset): def do_at_generic_first_appt( self, - patient_id: int, + person_id: int, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: # 'Automatic' testing for HIV for everyone attending care with AIDS symptoms: # - suppress the footprint (as it done as part of another appointment) # - do not do referrals if the person is HIV negative (assumed not time for counselling etc). if "aids_symptoms" in symptoms: event = HSI_Hiv_TestAndRefer( - person_id=patient_id, + person_id=person_id, module=self, referred_from="hsi_generic_first_appt", suppress_footprint=True, do_not_refer_if_neg=True, ) - self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date) + schedule_hsi_event(event, priority=0, topen=self.sim.date) # --------------------------------------------------------------------------- # Main Polling Event diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 5daa6e66f9..85feb2b1b5 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -127,13 +127,6 @@ def bed_days_allocated_to_this_event(self): return self._received_info_about_bed_days - @property - def target_is_alive(self) -> bool: - """Return True if the target of this HSI event is alive, - otherwise False. - """ - return self.sim.population.props.at[self.target, "is_alive"] - @property def sim(self) -> Simulation: return self.module.sim diff --git a/src/tlo/methods/hsi_generic_first_appts.py b/src/tlo/methods/hsi_generic_first_appts.py index 303255b81c..30f4d40ac7 100644 --- a/src/tlo/methods/hsi_generic_first_appts.py +++ b/src/tlo/methods/hsi_generic_first_appts.py @@ -1,146 +1,246 @@ -""" -The file contains the event HSI_GenericFirstApptAtFacilityLevel1, which describes the first interaction with -the health system following the onset of acute generic symptoms. +"""Events which describes the first interaction with the health system. -This file contains the HSI events that represent the first contact with the Health System, which are triggered by -the onset of symptoms. Non-emergency symptoms lead to `HSI_GenericFirstApptAtFacilityLevel0` and emergency symptoms -lead to `HSI_GenericEmergencyFirstApptAtFacilityLevel1`. +This module contains the HSI events that represent the first contact with the health +system, which are triggered by the onset of symptoms. Non-emergency symptoms lead to +:py:class:`HSI_GenericNonEmergencyFirstAppt` and emergency symptoms lead to +:py:class:`HSI_GenericEmergencyFirstAppt`. """ + from __future__ import annotations -from typing import TYPE_CHECKING, Literal, OrderedDict +from collections.abc import Callable +from typing import TYPE_CHECKING, Any, Dict, List, Protocol, Set, Union -from tlo import logging +import numpy as np + +from tlo import Date, Module, logging from tlo.events import IndividualScopeEventMixin from tlo.methods.hsi_event import HSI_Event if TYPE_CHECKING: - from tlo import Module + from typing import Optional, TypeAlias + from tlo.methods.dxmanager import DiagnosisTestReturnType + from tlo.population import IndividualProperties logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class HSI_BaseGenericFirstAppt(HSI_Event, IndividualScopeEventMixin): - """ - """ - MODULE_METHOD_ON_APPLY: Literal[ - "do_at_generic_first_appt", "do_at_generic_first_appt_emergency" - ] + +DiagnosisFunction: TypeAlias = Callable[[str, bool, bool], Any] +ConsumablesChecker: TypeAlias = Callable[ + [ + Union[None, np.integer, int, List, Set, Dict], + Union[None, np.integer, int, List, Set, Dict], + ], + Union[bool, Dict], +] + + +class HSIEventScheduler(Protocol): + + def __call__( + self, + hsi_event: HSI_Event, + priority: int, + topen: Date, + tclose: Optional[Date] = None, + ) -> None: ... + + +class GenericFirstAppointmentsMixin: + """Mix-in for modules with actions to perform on generic first appointments.""" + + def do_at_generic_first_appt( + self, + *, + person_id: int, + individual_properties: IndividualProperties, + symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, + diagnosis_function: DiagnosisFunction, + consumables_checker: ConsumablesChecker, + facility_level: str, + treatment_id: str, + ) -> None: + """ + Actions to take during a non-emergency generic health system interaction (HSI). + + Derived classes should overwrite this method so that they are compatible with + the :py:class:`~.HealthSystem` module, and can schedule HSI events when a + individual presents symptoms indicative of the corresponding illness or + condition. + + When overwriting, arguments that are not required can be left out of the + definition. If done so, the method **must** take a ``**kwargs`` input to avoid + errors when looping over all disease modules and running their generic HSI + methods. + + HSI events should be scheduled by the :py:class:`Module` subclass implementing + this method using the ``schedule_hsi_event`` argument. + + Implementations of this method should **not** make any updates to the population + dataframe directly - if the target individuals properties need to be updated + this should be performed by updating the ``individual_properties`` argument. + + :param person_id: Row index (ID) of the individual target of the HSI event in + the population dataframe. + :param individual_properties: Properties of individual target as provided in the + population dataframe. Updates to individual properties may be written to + this object. + :param symptoms: List of symptoms the patient is experiencing. + :param schedule_hsi_event: A function that can schedule subsequent HSI events. + :param diagnosis_function: A function that can run diagnosis tests based on the + patient's symptoms. + :param consumables_checker: A function that can query the health system to check + for available consumables. + :param facility_level: The level of the facility that the patient presented at. + :param treatment_id: The treatment id of the HSI event triggering the generic + appointment. + """ + + def do_at_generic_first_appt_emergency( + self, + *, + person_id: int, + individual_properties: IndividualProperties, + symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, + diagnosis_function: DiagnosisFunction, + consumables_checker: ConsumablesChecker, + facility_level: str, + treatment_id: str, + ) -> None: + """ + Actions to take during an emergency generic health system interaction (HSI). + + Call signature is identical to the + :py:meth:`~GenericFirstAppointmentsMixin.do_at_generic_first_appt` method. + + Derived classes should overwrite this method so that they are compatible with + the :py:class`~.HealthSystem` module, and can schedule HSI events when a + individual presents symptoms indicative of the corresponding illness or + condition. + """ + + +class _BaseHSIGenericFirstAppt(HSI_Event, IndividualScopeEventMixin): def __init__(self, module, person_id) -> None: super().__init__(module, person_id=person_id) - # No footprint, as this HSI (mostly just) determines which - # further HSI will be needed for this person. In some cases, - # small bits of care are provided (e.g. a diagnosis, or the - # provision of inhaler). - self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint( - {} - ) + # No footprint, as this HSI (mostly just) determines which further HSI will be + # needed for this person. In some cases, small bits of care are provided (e.g. a + # diagnosis, or the provision of inhaler). + self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({}) def _diagnosis_function( self, tests, use_dict: bool = False, report_tried: bool = False ) -> DiagnosisTestReturnType: """ - Passed to modules when determining HSI_Events to be scheduled based on - this generic appointment. Intended as the diagnosis_function argument to the - Module.do_at_generic_{non_}_emergency. + Passed to modules when determining HSI events to be scheduled based on + this generic appointment. Intended as the ``diagnosis_function`` argument to + :py:meth:`GenericFirstAppointmentsMixin.do_at_generic_first_appt` or + :py:meth:`GenericFirstAppointmentsMixin.do_at_generic_first_appt_emergency`. Class-level definition avoids the need to redefine this method each time - the .apply() method is called. + the :py:meth:`apply` method is called. :param tests: The name of the test(s) to run via the diagnosis manager. - :param use_dict_for_single: If True, the return type will be a dictionary - even if only one test was requested. + :param use_dict_for_single: If ``True``, the return type will be a dictionary + even if only one test was requested. :param report_dxtest_tried: Report if a test was attempted but could not - be carried out due to EG lack of consumables, etc. + be carried out due to for example lack of consumables, etc. :returns: Test results as dictionary key/value pairs. """ - return self.healthcare_system.dx_manager.run_dx_test( + return self.sim.modules["HealthSystem"].dx_manager.run_dx_test( tests, hsi_event=self, use_dict_for_single=use_dict, report_dxtest_tried=report_tried, ) - def _do_on_generic_first_appt(self, squeeze_factor: float = 0.) -> None: - """ + @staticmethod + def _do_at_generic_first_appt_for_module(module: Module) -> Callable: + """Retrieves relevant do_at_generic_first_appt* method for a module. + + Must be implemented by concrete classes derived from this base class. """ - # Make top-level reads of information, to avoid repeat accesses. - modules: OrderedDict[str, "Module"] = self.sim.modules - symptoms = modules["SymptomManager"].has_what(self.target) - - # Dynamically create immutable container with the target's details stored. - # This will avoid repeat DataFrame reads when we call the module-level functions. - patient_details = self.sim.population.row_in_readonly_form(self.target) - - for module in modules.values(): - module_patient_updates = getattr(module, self.MODULE_METHOD_ON_APPLY)( - patient_id=self.target, - patient_details=patient_details, - symptoms=symptoms, - diagnosis_function=self._diagnosis_function, - consumables_checker=self.get_consumables, - facility_level=self.ACCEPTED_FACILITY_LEVEL, - treatment_id=self.TREATMENT_ID, - random_state=self.module.rng, - ) - # Record any requested DataFrame updates - if module_patient_updates: - for key, value in module_patient_updates.items(): - self.sim.population.props.at[self.target, key] = value - # Also need to recreate patient_details to reflect updated properties - patient_details = self.sim.population.row_in_readonly_form(self.target) - - def apply(self, person_id, squeeze_factor=0.) -> None: + raise NotImplementedError + + def apply(self, person_id: int, squeeze_factor: float = 0.0) -> None: """ - Run the actions required during the HSI. + Run the actions required during the health system interaction (HSI). TODO: person_id is not needed any more - but would have to go through the whole codebase to manually identify instances of this class to change call syntax, and leave other HSI_Event-derived classes alone. """ - if self.target_is_alive: - self._do_on_generic_first_appt(squeeze_factor=squeeze_factor) + # Create a memoized view of target individuals' properties as a context manager + # that will automatically synchronize any updates back to the population + # dataframe on exit + with self.sim.population.individual_properties( + self.target, read_only=False + ) as individual_properties: + if not individual_properties["is_alive"]: + return + # Pre-evaluate symptoms for individual to avoid repeat accesses + # TODO: Use individual_properties to populate symptoms + symptoms = self.sim.modules["SymptomManager"].has_what(self.target) + schedule_hsi_event = self.sim.modules["HealthSystem"].schedule_hsi_event + for module in self.sim.modules.values(): + if isinstance(module, GenericFirstAppointmentsMixin): + self._do_at_generic_first_appt_for_module(module)( + person_id=self.target, + individual_properties=individual_properties, + symptoms=symptoms, + schedule_hsi_event=schedule_hsi_event, + diagnosis_function=self._diagnosis_function, + consumables_checker=self.get_consumables, + facility_level=self.ACCEPTED_FACILITY_LEVEL, + treatment_id=self.TREATMENT_ID, + ) + + +class HSI_GenericNonEmergencyFirstAppt(_BaseHSIGenericFirstAppt): + """ + This is a health system interaction event that represents the first interaction with + the health system following the onset of non-emergency symptom(s). + It is generated by the :py:class:`~HealthSeekingBehaviour` module. -class HSI_GenericNonEmergencyFirstAppt(HSI_BaseGenericFirstAppt): - """ - This is a Health System Interaction Event that represents the - first interaction with the health system following the onset - of non-emergency symptom(s). - - It is generated by the HealthSeekingBehaviour module. - - By default, it occurs at level '0' but it could occur also at - other levels. - - It uses the non-emergency generic first appointment methods of - the disease modules to determine any follow-up events that need - to be scheduled. + By default, it occurs at level '0' but it could occur also at other levels. + + It uses the non-emergency generic first appointment methods of the disease modules + to determine any follow-up events that need to be scheduled. """ - MODULE_METHOD_ON_APPLY = "do_at_generic_first_appt" - def __init__(self, module, person_id, facility_level='0'): - super().__init__(module, person_id=person_id, ) + def __init__(self, module, person_id, facility_level="0"): + super().__init__( + module, + person_id=person_id, + ) - assert module is self.sim.modules['HealthSeekingBehaviour'] + assert module is self.sim.modules["HealthSeekingBehaviour"] - self.TREATMENT_ID = 'FirstAttendance_NonEmergency' + self.TREATMENT_ID = "FirstAttendance_NonEmergency" self.ACCEPTED_FACILITY_LEVEL = facility_level + @staticmethod + def _do_at_generic_first_appt_for_module( + module: GenericFirstAppointmentsMixin, + ) -> Callable: + return module.do_at_generic_first_appt -class HSI_GenericEmergencyFirstAppt(HSI_BaseGenericFirstAppt): + +class HSI_GenericEmergencyFirstAppt(_BaseHSIGenericFirstAppt): """ - This is a Health System Interaction Event that represents - the generic appointment which is the first interaction with - the health system following the onset of emergency symptom(s). + This is a health system interaction event that represents the generic appointment + which is the first interaction with the health system following the onset of + emergency symptom(s). - It uses the emergency generic first appointment methods of - the disease modules to determine any follow-up events that need - to be scheduled. + It uses the emergency generic first appointment methods of the disease modules to + determine any follow-up events that need to be scheduled. """ - MODULE_METHOD_ON_APPLY = "do_at_generic_first_appt_emergency" def __init__(self, module, person_id): super().__init__(module, person_id=person_id) @@ -155,8 +255,15 @@ def __init__(self, module, person_id): self.TREATMENT_ID = "FirstAttendance_Emergency" self.ACCEPTED_FACILITY_LEVEL = "1b" + @staticmethod + def _do_at_generic_first_appt_for_module( + module: GenericFirstAppointmentsMixin, + ) -> Callable: + return module.do_at_generic_first_appt_emergency + + class HSI_EmergencyCare_SpuriousSymptom(HSI_Event, IndividualScopeEventMixin): - """This is an HSI event that provides Accident & Emergency Care for a person that has spurious emergency symptom.""" + """HSI event providing accident & emergency care on spurious emergency symptoms.""" def __init__(self, module, person_id, accepted_facility_level="1a"): super().__init__(module, person_id=person_id) diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 2280d41335..c924f017cc 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -8,20 +8,19 @@ import scipy.stats from tlo import Date, DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType from tlo.methods import Metadata, labour_lm, pregnancy_helper_functions from tlo.methods.causes import Cause from tlo.methods.dxmanager import DxTest from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.postnatal_supervisor import PostnatalWeekOneMaternalEvent from tlo.util import BitsetHandler if TYPE_CHECKING: - from numpy.random import RandomState - - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + from tlo.population import IndividualProperties # Standard logger @@ -37,7 +36,7 @@ logger_pn.setLevel(logging.INFO) -class Labour(Module): +class Labour(Module, GenericFirstAppointmentsMixin): """This is module is responsible for the process of labour, birth and the immediate postnatal period (up until 48hrs post birth). This model has a number of core functions including; initiating the onset of labour for women on their pre-determined due date (or prior to this for preterm labour/admission for delivery), applying the incidence @@ -2323,28 +2322,27 @@ def run_if_receives_comprehensive_emergency_obstetric_care_cant_run(self, hsi_ev def do_at_generic_first_appt_emergency( self, - patient_id: int, - patient_details: PatientDetails, - random_state: RandomState, + person_id: int, + individual_properties: IndividualProperties, + schedule_hsi_event: HSIEventScheduler, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: mni = self.sim.modules["PregnancySupervisor"].mother_and_newborn_info labour_list = self.sim.modules["Labour"].women_in_labour - if patient_id in labour_list: - la_currently_in_labour = patient_details.la_currently_in_labour + if person_id in labour_list: + la_currently_in_labour = individual_properties["la_currently_in_labour"] if ( la_currently_in_labour - & mni[patient_id]["sought_care_for_complication"] - & (mni[patient_id]["sought_care_labour_phase"] == "intrapartum") + & mni[person_id]["sought_care_for_complication"] + & (mni[person_id]["sought_care_labour_phase"] == "intrapartum") ): event = HSI_Labour_ReceivesSkilledBirthAttendanceDuringLabour( module=self, - person_id=patient_id, - # facility_level_of_this_hsi=random_state.choice(["1a", "1b"]), + person_id=person_id, facility_level_of_this_hsi=self.rng.choice(["1a", "1b"]), ) - self.healthsystem.schedule_hsi_event( + schedule_hsi_event( event, priority=0, topen=self.sim.date, diff --git a/src/tlo/methods/malaria.py b/src/tlo/methods/malaria.py index 7731ea923c..50ad58db8b 100644 --- a/src/tlo/methods/malaria.py +++ b/src/tlo/methods/malaria.py @@ -12,24 +12,25 @@ import pandas as pd from tlo import DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import DiagnosisFunction, IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, Predictor from tlo.methods import Metadata from tlo.methods.causes import Cause from tlo.methods.dxmanager import DxTest from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom from tlo.util import random_date if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import DiagnosisFunction, HSIEventScheduler + from tlo.population import IndividualProperties logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class Malaria(Module): +class Malaria(Module, GenericFirstAppointmentsMixin): def __init__(self, name=None, resourcefilepath=None): """Create instance of Malaria module @@ -675,7 +676,7 @@ def check_if_fever_is_caused_by_malaria( self, true_malaria_infection_type: str, diagnosis_function: DiagnosisFunction, - patient_id: Optional[int] = None, + person_id: Optional[int] = None, fever_is_a_symptom: Optional[bool] = True, patient_age: Optional[Union[int, float]] = None, facility_level: Optional[str] = None, @@ -695,7 +696,7 @@ def check_if_fever_is_caused_by_malaria( logger.info( key="rdt_log", data={ - "person_id": patient_id, + "person_id": person_id, "age": patient_age, "fever_present": fever_is_a_symptom, "rdt_result": dx_result, @@ -714,16 +715,15 @@ def check_if_fever_is_caused_by_malaria( def do_at_generic_first_appt( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, diagnosis_function: DiagnosisFunction, facility_level: str, treatment_id: str, **kwargs, - ) -> IndividualPropertyUpdates: - patient_details_updates = {} - + ) -> None: malaria_associated_symptoms = { "fever", "headache", @@ -733,75 +733,72 @@ def do_at_generic_first_appt( } if ( bool(set(symptoms) & malaria_associated_symptoms) - and patient_details.ma_tx == "none" + and individual_properties["ma_tx"] == "none" ): malaria_test_result = self.check_if_fever_is_caused_by_malaria( - true_malaria_infection_type=patient_details.ma_inf_type, + true_malaria_infection_type=individual_properties["ma_inf_type"], diagnosis_function=diagnosis_function, - patient_id=patient_id, + person_id=person_id, fever_is_a_symptom="fever" in symptoms, - patient_age=patient_details.age_years, + patient_age=individual_properties["age_years"], facility_level=facility_level, treatment_id=treatment_id, ) # Treat / refer based on diagnosis if malaria_test_result == "severe_malaria": - patient_details_updates["ma_dx_counter"] = patient_details.ma_dx_counter + 1 - event = HSI_Malaria_Treatment_Complicated(person_id=patient_id, module=self) - self.healthsystem.schedule_hsi_event( + individual_properties["ma_dx_counter"] += 1 + event = HSI_Malaria_Treatment_Complicated(person_id=person_id, module=self) + schedule_hsi_event( event, priority=0, topen=self.sim.date ) # return type 'clinical_malaria' includes asymptomatic infection elif malaria_test_result == "clinical_malaria": - patient_details_updates["ma_dx_counter"] = patient_details.ma_dx_counter + 1 - event = HSI_Malaria_Treatment(person_id=patient_id, module=self) - self.healthsystem.schedule_hsi_event( + individual_properties["ma_dx_counter"] += 1 + event = HSI_Malaria_Treatment(person_id=person_id, module=self) + schedule_hsi_event( event, priority=1, topen=self.sim.date ) - return patient_details_updates def do_at_generic_first_appt_emergency( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, diagnosis_function: DiagnosisFunction, facility_level: str, treatment_id: str, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: # This is called for a person (of any age) that attends an # emergency generic HSI and has a fever. # (Quick diagnosis algorithm - just perfectly recognises the # symptoms of severe malaria.) - patient_details_updates = {} - if 'severe_malaria' in symptoms: - if patient_details.ma_tx == 'none': + if individual_properties["ma_tx"] == 'none': # Check if malaria parasitaemia: malaria_test_result = self.check_if_fever_is_caused_by_malaria( - true_malaria_infection_type=patient_details.ma_inf_type, + true_malaria_infection_type=individual_properties["ma_inf_type"], diagnosis_function=diagnosis_function, - patient_id=patient_id, + person_id=person_id, fever_is_a_symptom="fever" in symptoms, - patient_age=patient_details.age_years, + patient_age=individual_properties["age_years"], facility_level=facility_level, treatment_id=treatment_id, ) # if any symptoms indicative of malaria and they have parasitaemia (would return a positive rdt) if malaria_test_result in ('severe_malaria', 'clinical_malaria'): - patient_details_updates['ma_dx_counter'] = patient_details.ma_dx_counter + 1 + individual_properties['ma_dx_counter'] += 1 # Launch the HSI for treatment for Malaria, HSI_Malaria_Treatment will determine correct treatment event = HSI_Malaria_Treatment_Complicated( - person_id=patient_id, module=self, + person_id=person_id, module=self, ) - self.healthsystem.schedule_hsi_event( + schedule_hsi_event( event, priority=0, topen=self.sim.date ) - return patient_details_updates class MalariaPollingEventDistrict(RegularEvent, PopulationScopeEventMixin): """ diff --git a/src/tlo/methods/measles.py b/src/tlo/methods/measles.py index 01481a060c..5de96c9883 100644 --- a/src/tlo/methods/measles.py +++ b/src/tlo/methods/measles.py @@ -1,23 +1,28 @@ +from __future__ import annotations + import math import os -from typing import List +from typing import TYPE_CHECKING, List import pandas as pd from tlo import DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.methods import Metadata from tlo.methods.causes import Cause from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom from tlo.util import random_date +if TYPE_CHECKING: + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class Measles(Module): +class Measles(Module, GenericFirstAppointmentsMixin): """This module represents measles infections and disease.""" INIT_DEPENDENCIES = {'Demography', 'HealthSystem', 'SymptomManager'} @@ -207,13 +212,14 @@ def process_parameters(self): def do_at_generic_first_appt( self, - patient_id: int, + person_id: int, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: if "rash" in symptoms: - event = HSI_Measles_Treatment(person_id=patient_id, module=self) - self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date) + event = HSI_Measles_Treatment(person_id=person_id, module=self) + schedule_hsi_event(event, priority=0, topen=self.sim.date) class MeaslesEvent(RegularEvent, PopulationScopeEventMixin): diff --git a/src/tlo/methods/mockitis.py b/src/tlo/methods/mockitis.py index e349dd188e..6af33c5fc7 100644 --- a/src/tlo/methods/mockitis.py +++ b/src/tlo/methods/mockitis.py @@ -1,21 +1,26 @@ -from typing import List +from __future__ import annotations + +from typing import TYPE_CHECKING, List import pandas as pd from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.methods import Metadata from tlo.methods.causes import Cause from tlo.methods.demography import InstantaneousDeath from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom +if TYPE_CHECKING: + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class Mockitis(Module): +class Mockitis(Module, GenericFirstAppointmentsMixin): """This is a dummy infectious disease. It demonstrates the following behaviours in respect of the healthsystem module: @@ -290,17 +295,18 @@ def report_daly_values(self): def do_at_generic_first_appt_emergency( self, - patient_id: int, + person_id: int, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: # Example for mockitis if "extreme_pain_in_the_nose" in symptoms: event = HSI_Mockitis_PresentsForCareWithSevereSymptoms( module=self, - person_id=patient_id, + person_id=person_id, ) - self.healthsystem.schedule_hsi_event(event, priority=1, topen=self.sim.date) + schedule_hsi_event(event, priority=1, topen=self.sim.date) class MockitisEvent(RegularEvent, PopulationScopeEventMixin): """ diff --git a/src/tlo/methods/oesophagealcancer.py b/src/tlo/methods/oesophagealcancer.py index b3a302bcd9..104770a15f 100644 --- a/src/tlo/methods/oesophagealcancer.py +++ b/src/tlo/methods/oesophagealcancer.py @@ -14,7 +14,6 @@ import pandas as pd from tlo import DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata @@ -23,16 +22,18 @@ from tlo.methods.demography import InstantaneousDeath from tlo.methods.dxmanager import DxTest from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + from tlo.population import IndividualProperties logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class OesophagealCancer(Module): +class OesophagealCancer(Module, GenericFirstAppointmentsMixin): """Oesophageal Cancer Disease Module""" def __init__(self, name=None, resourcefilepath=None): @@ -578,18 +579,19 @@ def report_daly_values(self): def do_at_generic_first_appt( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: # If the symptoms include dysphagia, and the patient is not a child, # begin investigation for Oesophageal Cancer: - if patient_details.age_years > 5 and "dysphagia" in symptoms: + if individual_properties["age_years"] > 5 and "dysphagia" in symptoms: event = HSI_OesophagealCancer_Investigation_Following_Dysphagia( - person_id=patient_id, module=self + person_id=person_id, module=self ) - self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date) + schedule_hsi_event(event, priority=0, topen=self.sim.date) # --------------------------------------------------------------------------------------------------------- diff --git a/src/tlo/methods/other_adult_cancers.py b/src/tlo/methods/other_adult_cancers.py index 3bea8933e6..774b809cfc 100644 --- a/src/tlo/methods/other_adult_cancers.py +++ b/src/tlo/methods/other_adult_cancers.py @@ -12,7 +12,6 @@ import pandas as pd from tlo import DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata @@ -21,16 +20,18 @@ from tlo.methods.demography import InstantaneousDeath from tlo.methods.dxmanager import DxTest from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + from tlo.population import IndividualProperties logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class OtherAdultCancer(Module): +class OtherAdultCancer(Module, GenericFirstAppointmentsMixin): """Other Adult Cancers Disease Module""" def __init__(self, name=None, resourcefilepath=None): @@ -576,17 +577,18 @@ def report_daly_values(self): def do_at_generic_first_appt( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs - ) -> IndividualPropertyUpdates: - if patient_details.age_years > 5 and "early_other_adult_ca_symptom" in symptoms: + ) -> None: + if individual_properties["age_years"] > 5 and "early_other_adult_ca_symptom" in symptoms: event = HSI_OtherAdultCancer_Investigation_Following_early_other_adult_ca_symptom( - person_id=patient_id, + person_id=person_id, module=self, ) - self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date) + schedule_hsi_event(event, priority=0, topen=self.sim.date) # --------------------------------------------------------------------------------------------------------- diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py index 7d89181344..7dd8819ab6 100644 --- a/src/tlo/methods/pregnancy_supervisor.py +++ b/src/tlo/methods/pregnancy_supervisor.py @@ -18,7 +18,6 @@ logging, util, ) -from tlo.core import IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel from tlo.methods import Metadata, labour, pregnancy_helper_functions, pregnancy_supervisor_lm @@ -27,16 +26,18 @@ HSI_CareOfWomenDuringPregnancy_TreatmentForEctopicPregnancy, ) from tlo.methods.causes import Cause +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.util import BitsetHandler if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + from tlo.population import IndividualProperties logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class PregnancySupervisor(Module): +class PregnancySupervisor(Module, GenericFirstAppointmentsMixin): """This module is responsible for simulating the antenatal period of pregnancy (the period from conception until the termination of pregnancy). A number of outcomes are managed by this module including early pregnancy loss (induced/spontaneous abortion, ectopic pregnancy and antenatal stillbirth) and pregnancy complications of the @@ -1673,10 +1674,11 @@ def schedule_late_visit(df_slice): def do_at_generic_first_appt_emergency( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, + schedule_hsi_event: HSIEventScheduler, **kwargs, - ) -> IndividualPropertyUpdates: + ) -> None: scheduling_options = { "priority": 0, "topen": self.sim.date, @@ -1684,25 +1686,25 @@ def do_at_generic_first_appt_emergency( } # ----- ECTOPIC PREGNANCY ----- - if patient_details.ps_ectopic_pregnancy != 'none': + if individual_properties["ps_ectopic_pregnancy"] != 'none': event = HSI_CareOfWomenDuringPregnancy_TreatmentForEctopicPregnancy( module=self.sim.modules["CareOfWomenDuringPregnancy"], - person_id=patient_id, + person_id=person_id, ) - self.healthsystem.schedule_hsi_event(event, **scheduling_options) + schedule_hsi_event(event, **scheduling_options) # ----- COMPLICATIONS OF ABORTION ----- abortion_complications = self.sim.modules[ "PregnancySupervisor" ].abortion_complications if abortion_complications.has_any( - [patient_id], "sepsis", "injury", "haemorrhage", first=True + [person_id], "sepsis", "injury", "haemorrhage", first=True ): event = HSI_CareOfWomenDuringPregnancy_PostAbortionCaseManagement( module=self.sim.modules["CareOfWomenDuringPregnancy"], - person_id=patient_id, + person_id=person_id, ) - self.healthsystem.schedule_hsi_event(event, **scheduling_options) + schedule_hsi_event(event, **scheduling_options) class PregnancySupervisorEvent(RegularEvent, PopulationScopeEventMixin): """ This is the PregnancySupervisorEvent, it is a weekly event which has four primary functions. diff --git a/src/tlo/methods/prostate_cancer.py b/src/tlo/methods/prostate_cancer.py index f9520052b1..dabb2e0593 100644 --- a/src/tlo/methods/prostate_cancer.py +++ b/src/tlo/methods/prostate_cancer.py @@ -12,7 +12,6 @@ import pandas as pd from tlo import DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata @@ -21,16 +20,18 @@ from tlo.methods.demography import InstantaneousDeath from tlo.methods.dxmanager import DxTest from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + from tlo.population import IndividualProperties logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class ProstateCancer(Module): +class ProstateCancer(Module, GenericFirstAppointmentsMixin): """Prostate Cancer Disease Module""" def __init__(self, name=None, resourcefilepath=None): @@ -590,29 +591,30 @@ def report_daly_values(self): def do_at_generic_first_appt( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs - ) -> IndividualPropertyUpdates: + ) -> None: # If the patient is not a child, and symptoms are indicative, # begin investigation for prostate cancer scheduling_options = { "priority": 0, "topen": self.sim.date, } - if patient_details.age_years > 5: + if individual_properties["age_years"] > 5: if "urinary" in symptoms: event = HSI_ProstateCancer_Investigation_Following_Urinary_Symptoms( - person_id=patient_id, module=self + person_id=person_id, module=self ) - self.healthsystem.schedule_hsi_event(event, **scheduling_options) + schedule_hsi_event(event, **scheduling_options) if "pelvic_pain" in symptoms: event = HSI_ProstateCancer_Investigation_Following_Pelvic_Pain( - person_id=patient_id, module=self + person_id=person_id, module=self ) - self.healthsystem.schedule_hsi_event(event, **scheduling_options) + schedule_hsi_event(event, **scheduling_options) # --------------------------------------------------------------------------------------------------------- diff --git a/src/tlo/methods/rti.py b/src/tlo/methods/rti.py index f591f5aa7b..d6aa1d693f 100644 --- a/src/tlo/methods/rti.py +++ b/src/tlo/methods/rti.py @@ -11,16 +11,17 @@ import pandas as pd from tlo import DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata from tlo.methods.causes import Cause from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + from tlo.population import IndividualProperties # --------------------------------------------------------------------------------------------------------- # MODULE DEFINITIONS @@ -30,7 +31,7 @@ logger.setLevel(logging.DEBUG) -class RTI(Module): +class RTI(Module, GenericFirstAppointmentsMixin): """ The road traffic injuries module for the TLO model, handling all injuries related to road traffic accidents. """ @@ -2494,33 +2495,33 @@ def rti_assign_injuries(self, number): def _common_first_appt_steps( self, - patient_id: int, - patient_details: PatientDetails, - ) -> IndividualPropertyUpdates: + person_id: int, + individual_properties: IndividualProperties, + schedule_hsi_event: HSIEventScheduler, + ) -> None: """ Shared logic steps that are used by the RTI module when a generic HSI event is to be scheduled. """ - patient_details_updates = {} # Things to do upon a person presenting at a Non-Emergency Generic # HSI if they have an injury. persons_injuries = [ - getattr(patient_details, injury) for injury in RTI.INJURY_COLUMNS + individual_properties[injury] for injury in RTI.INJURY_COLUMNS ] if ( - pd.isnull(patient_details.cause_of_death) - and not patient_details.rt_diagnosed + pd.isnull(individual_properties["cause_of_death"]) + and not individual_properties["rt_diagnosed"] ): if set(RTI.INJURIES_REQ_IMAGING).intersection(set(persons_injuries)): - if patient_details.is_alive: - event = HSI_RTI_Imaging_Event(module=self, person_id=patient_id) - self.healthsystem.schedule_hsi_event( + if individual_properties["is_alive"]: + event = HSI_RTI_Imaging_Event(module=self, person_id=person_id) + schedule_hsi_event( event, priority=0, topen=self.sim.date + DateOffset(days=1), tclose=self.sim.date + DateOffset(days=15), ) - patient_details_updates["rt_diagnosed"] = True + individual_properties["rt_diagnosed"] = True # The injured person has been diagnosed in A&E and needs to progress further # through the health system. @@ -2546,47 +2547,56 @@ def _common_first_appt_steps( # If they meet the requirements, send them to HSI_RTI_MedicalIntervention for further treatment # Using counts condition to stop spurious symptoms progressing people through the model if counts > 0: - event = HSI_RTI_Medical_Intervention(module=self, person_id=patient_id) - self.healthsystem.schedule_hsi_event( - event, priority=0, topen=self.sim.date, + event = HSI_RTI_Medical_Intervention(module=self, person_id=person_id) + schedule_hsi_event( + event, + priority=0, + topen=self.sim.date, ) # We now check if they need shock treatment - if patient_details.rt_in_shock and patient_details.is_alive: - event = HSI_RTI_Shock_Treatment(module=self, person_id=patient_id) - self.healthsystem.schedule_hsi_event( + if ( + individual_properties["rt_in_shock"] + and individual_properties["is_alive"] + ): + event = HSI_RTI_Shock_Treatment(module=self, person_id=person_id) + schedule_hsi_event( event, priority=0, topen=self.sim.date + DateOffset(days=1), tclose=self.sim.date + DateOffset(days=15), ) - return patient_details_updates def do_at_generic_first_appt( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs - ) -> IndividualPropertyUpdates: + ) -> None: if "injury" in symptoms: return self._common_first_appt_steps( - patient_id=patient_id, patient_details=patient_details + person_id=person_id, + individual_properties=individual_properties, + schedule_hsi_event=schedule_hsi_event, ) def do_at_generic_first_appt_emergency( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs - ) -> IndividualPropertyUpdates: + ) -> None: # Same process is followed for emergency and non emergency appointments, except the # initial symptom check if "severe_trauma" in symptoms: return self._common_first_appt_steps( - patient_id=patient_id, - patient_details=patient_details + person_id=person_id, + individual_properties=individual_properties, + schedule_hsi_event=schedule_hsi_event, ) diff --git a/src/tlo/methods/schisto.py b/src/tlo/methods/schisto.py index 8ccc593601..810c869f10 100644 --- a/src/tlo/methods/schisto.py +++ b/src/tlo/methods/schisto.py @@ -1,19 +1,24 @@ +from __future__ import annotations + from pathlib import Path -from typing import List, Optional, Sequence, Union +from typing import TYPE_CHECKING, List, Optional, Sequence, Union import numpy as np import pandas as pd from tlo import Date, DateOffset, Module, Parameter, Property, Types, logging from tlo.analysis.utils import flatten_multi_index_series_into_dict_for_logging -from tlo.core import IndividualPropertyUpdates from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.methods import Metadata from tlo.methods.causes import Cause from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin from tlo.methods.symptommanager import Symptom from tlo.util import random_date +if TYPE_CHECKING: + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -26,7 +31,7 @@ _AGE_GROUPS = {'PSAC': (0, 4), 'SAC': (5, 14), 'Adults': (15, 120), 'All': (0, 120)} -class Schisto(Module): +class Schisto(Module, GenericFirstAppointmentsMixin): """Schistosomiasis module. Two species of worm that cause Schistosomiasis are modelled independently. Worms are acquired by persons via the environment. There is a delay between the acquisition of worms and the maturation to 'adults' worms; and a long @@ -326,19 +331,20 @@ def _schedule_mda_events(self) -> None: def do_at_generic_first_appt( self, - patient_id: int, + person_id: int, symptoms: List[str], + schedule_hsi_event: HSIEventScheduler, **kwargs - ) -> IndividualPropertyUpdates: + ) -> None: # Do when person presents to the GenericFirstAppt. # If the person has certain set of symptoms, refer ta HSI for testing. set_of_symptoms_indicative_of_schisto = {'anemia', 'haematuria', 'bladder_pathology'} if set_of_symptoms_indicative_of_schisto.issubset(symptoms): event = HSI_Schisto_TestingFollowingSymptoms( - module=self, person_id=patient_id + module=self, person_id=person_id ) - self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date) + schedule_hsi_event(event, priority=0, topen=self.sim.date) class SchistoSpecies: diff --git a/src/tlo/methods/stunting.py b/src/tlo/methods/stunting.py index bacb9f4d7d..002d24bc31 100644 --- a/src/tlo/methods/stunting.py +++ b/src/tlo/methods/stunting.py @@ -19,14 +19,15 @@ from scipy.stats import norm from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging -from tlo.core import IndividualPropertyUpdates from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata from tlo.methods.hsi_event import HSI_Event +from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin if TYPE_CHECKING: - from tlo.population import PatientDetails + from tlo.methods.hsi_generic_first_appts import HSIEventScheduler + from tlo.population import IndividualProperties logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -36,7 +37,7 @@ # MODULE DEFINITION # --------------------------------------------------------------------------------------------------------- -class Stunting(Module): +class Stunting(Module, GenericFirstAppointmentsMixin): """This is the disease module for Stunting""" INIT_DEPENDENCIES = {'Demography', 'Wasting', 'NewbornOutcomes', 'Diarrhoea', 'Hiv'} @@ -287,30 +288,31 @@ def do_treatment(self, person_id, prob_success): def do_at_generic_first_appt( self, - patient_id: int, - patient_details: PatientDetails, + person_id: int, + individual_properties: IndividualProperties, + schedule_hsi_event: HSIEventScheduler, **kwargs, - ) -> IndividualPropertyUpdates: - # This is called by the a generic HSI event for every child aged - # less than 5 years. - # It assesses stunting and schedules an HSI as needed. - is_stunted = patient_details.un_HAZ_category in ('HAZ<-3', '-3<=HAZ<-2') - p_stunting_diagnosed = self.parameters['prob_stunting_diagnosed_at_generic_appt'] - - # Schedule the HSI for provision of treatment based on the - # probability of stunting diagnosis, provided the necessary - # symptoms are there. - if ( - (patient_details.age_years <= 5) - and is_stunted - ): - # Schedule the HSI for provision of treatment based on the - # probability of stunting diagnosis + ) -> None: + # This is called by the a generic HSI event for every child aged less than 5 + # years. It assesses stunting and schedules an HSI as needed. + is_stunted = individual_properties["un_HAZ_category"] in ( + "HAZ<-3", + "-3<=HAZ<-2", + ) + p_stunting_diagnosed = self.parameters[ + "prob_stunting_diagnosed_at_generic_appt" + ] + + # Schedule the HSI for provision of treatment based on the probability of + # stunting diagnosis, provided the necessary symptoms are there. + if individual_properties["age_years"] <= 5 and is_stunted: + # Schedule the HSI for provision of treatment based on the probability of + # stunting diagnosis if p_stunting_diagnosed > self.rng.random_sample(): event = HSI_Stunting_ComplementaryFeeding( - module=self, person_id=patient_id + module=self, person_id=person_id ) - self.healthsystem.schedule_hsi_event( + schedule_hsi_event( event, priority=2, # <-- lower priority that for wasting and most other HSI topen=self.sim.date, diff --git a/src/tlo/population.py b/src/tlo/population.py index 74ae0692e7..37f5fccfdf 100644 --- a/src/tlo/population.py +++ b/src/tlo/population.py @@ -1,39 +1,81 @@ -"""The Person and Population classes.""" +"""Types for representing a properties of a population of individuals.""" import math -from typing import Any, Dict +from collections.abc import Generator +from contextlib import contextmanager +from typing import Any, Dict, Optional, Set import pandas as pd -from tlo import logging +from tlo import Property, logging logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -class PatientDetails: - """Read-only memoized view of population dataframe row.""" - - def __init__(self, population_dataframe: pd.DataFrame, person_id: int): - self._population_dataframe = population_dataframe - self._person_id = person_id +class IndividualProperties: + """Memoized view of population dataframe row that is optionally read-only. + + This class should not be instantiated directly but instead the + :py:meth:`Population.individual_properties` context manager method used to create + instances for a given population. + """ + + def __init__( + self, population_dataframe: pd.DataFrame, person_id: int, read_only: bool = True + ): + self._finalized = False + self._read_only = read_only self._property_cache: Dict[str, Any] = {} - + # Avoid storing a reference to population_dataframe internally by mediating + # access via closures to guard against direct access + self._get_value_at = lambda key: population_dataframe.at[person_id, key] + if not read_only: + self._properties_updated: Set[str] = set() + + def synchronize_updates_to_dataframe(): + row_index = population_dataframe.index.get_loc(person_id) + for key in self._properties_updated: + # This chained indexing approach to setting dataframe values is + # significantly (~3 to 4 times) quicker than using at / iat + # indexers, but will fail when copy-on-write is enabled which will + # be default in Pandas 3 + column = population_dataframe[key] + column.values[row_index] = self._property_cache[key] + + self._synchronize_updates_to_dataframe = synchronize_updates_to_dataframe + def __getitem__(self, key: str) -> Any: + if self._finalized: + msg = f"Cannot read value for {key} as instance has been finalized" + raise ValueError(msg) try: return self._property_cache[key] except KeyError: - value = self._population_dataframe.at[self._person_id, key] + value = self._get_value_at(key) self._property_cache[key] = value - return value - - def __getattr__(self, name: str) -> Any: - try: - return self[name] - except KeyError as e: - msg = f"'{type(self).__name__}' object has no attribute '{name}'" - raise AttributeError(msg) from e - + return value + + def __setitem__(self, key: str, value: Any) -> None: + if self._finalized: + msg = f"Cannot set value for {key} as instance has been finalized" + raise ValueError(msg) + if self._read_only: + msg = f"Cannot set value for {key} as destination is read-only" + raise ValueError(msg) + self._properties_updated.add(key) + self._property_cache[key] = value + + def synchronize_updates_to_dataframe(self) -> None: + """Synchronize values for any updated properties to population dataframe.""" + if not self._read_only: + self._synchronize_updates_to_dataframe() + self._properties_updated.clear() + + def finalize(self) -> None: + """Synchronize updates to population dataframe and prevent further access.""" + self.synchronize_updates_to_dataframe() + self._finalized = True class Population: @@ -41,39 +83,41 @@ class Population: Useful properties of a population: - `sim` - The Simulation instance controlling this population. - `props` A Pandas DataFrame with the properties of all individuals as columns. """ __slots__ = ( - "_patient_details_readonly_type", "props", - "sim", "initial_size", "new_row", "next_person_id", "new_rows", ) - def __init__(self, sim, initial_size: int, append_size: int = None): + def __init__( + self, + properties: Dict[str, Property], + initial_size: int, + append_size: Optional[int] = None, + ): """Create a new population. - This will create the required the population dataframe and initialise individual's - properties as dataframe columns with 'empty' values. The simulation will then call disease - modules to fill in suitable starting values. + This will create the required the population dataframe and initialise + individual's properties as dataframe columns with 'empty' values. The simulation + will then call disease modules to fill in suitable starting values. - :param sim: the Simulation containing this population - :param initial_size: the initial population size - :param append_size: how many rows to append when growing the population dataframe (optional) + :param properties: Dictionary defining properties (columns) to initialise + population dataframe with, keyed by property name and with values + :py:class:`Property` instances defining the property type. + :param initial_size: The initial population size. + :param append_size: How many rows to append when growing the population + dataframe (optional). """ - self.sim = sim self.initial_size = initial_size # Create empty property arrays - self.props = self._create_props(initial_size) + self.props = self._create_props(initial_size, properties) if append_size is None: # approximation based on runs to increase capacity of dataframe ~twice a year @@ -93,7 +137,7 @@ def __init__(self, sim, initial_size: int, append_size: int = None): # use the person_id of the next person to be added to the dataframe to increase capacity self.next_person_id = initial_size - def _create_props(self, size): + def _create_props(self, size: int, properties: Dict[str, Property]) -> pd.DataFrame: """Internal helper function to create a properties dataframe. :param size: the number of rows to create @@ -101,8 +145,7 @@ def _create_props(self, size): return pd.DataFrame( data={ property_name: property.create_series(property_name, size) - for module in self.sim.modules.values() - for property_name, property in module.PROPERTIES.items() + for property_name, property in properties.items() }, index=pd.RangeIndex(stop=size, name="person"), ) @@ -154,15 +197,33 @@ def make_test_property(self, name, type_): size = self.initial_size if self.props.empty else len(self.props) self.props[name] = prop.create_series(name, size) - def row_in_readonly_form(self, patient_index: int) -> PatientDetails: + @contextmanager + def individual_properties( + self, person_id: int, read_only: bool = True + ) -> Generator[IndividualProperties, None, None]: """ - Extract a lazily evaluated, read-only view of a row of the population dataframe. - - The object returned represents the properties of an individual with properties - accessible either using dot based attribute access or squared bracket based - indexing using string column names. + Context manager for a memoized view of a row of the population dataframe. - :param patient_index: Row index of the dataframe row to extract. - :returns: Object allowing read-only access to an individuals properties. + The view returned represents the properties of an individual with properties + accessible by indexing using string column names, and lazily read-on demand + from the population dataframe. + + Optionally the view returned may allow updating properties as well as reading. + In this case on exit from the ``with`` block in which the context is entered, + any updates to the individual properties will be written back to the population + dataframe. + + Once the ``with`` block in which the context is entered has been exited the view + returned will raise an error on any subsequent attempts at reading or writing + properties. + + :param person_id: Row index of the dataframe row to extract. + :param read_only: Whether view is read-only or allows updating properties. If + ``True`` :py:meth:`IndividualProperties.synchronize_updates_to_dataframe` + method needs to be called for any updates to be written back to population + dataframe. + :returns: Object allowing memoized access to an individual's properties. """ - return PatientDetails(self.props, patient_index) + properties = IndividualProperties(self.props, person_id, read_only=read_only) + yield properties + properties.finalize() diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 544ae0d68f..761c161799 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -183,7 +183,12 @@ def make_initial_population(self, *, n): module.pre_initialise_population() # Make the initial population - self.population = Population(self, n) + properties = { + name: prop + for module in self.modules.values() + for name, prop in module.PROPERTIES.items() + } + self.population = Population(properties, n) for module in self.modules.values(): start1 = time.time() module.initialise_population(self.population) diff --git a/src/tlo/test/random_birth.py b/src/tlo/test/random_birth.py index 950173797d..22a20879b1 100644 --- a/src/tlo/test/random_birth.py +++ b/src/tlo/test/random_birth.py @@ -68,7 +68,7 @@ def initialise_population(self, population): # We use 'broadcasting' to set the same value for every individual df.is_pregnant = False # We randomly sample birth dates for the initial population during the preceding decade - start_date = population.sim.date + start_date = self.sim.date dates = pd.date_range(start_date - DateOffset(years=10), start_date, freq='M') df.date_of_birth = self.rng.choice(dates, size=len(df)) # No children have yet been born. We iterate over the population to ensure each diff --git a/tests/test_basic_sims.py b/tests/test_basic_sims.py index 5200ab730c..ea29aa601f 100644 --- a/tests/test_basic_sims.py +++ b/tests/test_basic_sims.py @@ -184,7 +184,7 @@ def __init__(self, module, end_date): super().__init__(module=module, frequency=DateOffset(days=1), end_date=end_date) def apply(self, population): - population.props.loc[0, 'last_run'] = population.sim.date + population.props.loc[0, 'last_run'] = self.module.sim.date class MyOtherEvent(PopulationScopeEventMixin, RegularEvent): def __init__(self, module): diff --git a/tests/test_diarrhoea.py b/tests/test_diarrhoea.py index dc3330c225..3a4daebc5d 100644 --- a/tests/test_diarrhoea.py +++ b/tests/test_diarrhoea.py @@ -403,7 +403,6 @@ def test_do_when_presentation_with_diarrhoea_severe_dehydration(seed): generic_hsi = HSI_GenericNonEmergencyFirstAppt( module=sim.modules["HealthSeekingBehaviour"], person_id=person_id ) - patient_details = sim.population.row_in_readonly_form(person_id) symptoms = {"diarrhoea"} def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False): @@ -416,12 +415,14 @@ def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False): sim.modules['HealthSystem'].reset_queue() sim.modules['Diarrhoea'].parameters['prob_hospitalization_on_danger_signs'] = 1.0 - sim.modules["Diarrhoea"].do_at_generic_first_appt( - patient_id=person_id, - patient_details=patient_details, - diagnosis_function=diagnosis_fn, - symptoms=symptoms, - ) + with sim.population.individual_properties(person_id) as individual_properties: + sim.modules["Diarrhoea"].do_at_generic_first_appt( + person_id=person_id, + individual_properties=individual_properties, + schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event, + diagnosis_function=diagnosis_fn, + symptoms=symptoms, + ) evs = sim.modules['HealthSystem'].find_events_for_person(person_id) assert 1 == len(evs) @@ -430,12 +431,14 @@ def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False): # 2) If DxTest of danger signs perfect but 0% chance of referral --> Inpatient HSI should not be created sim.modules['HealthSystem'].reset_queue() sim.modules['Diarrhoea'].parameters['prob_hospitalization_on_danger_signs'] = 0.0 - sim.modules["Diarrhoea"].do_at_generic_first_appt( - patient_id=person_id, - patient_details=patient_details, - diagnosis_function=diagnosis_fn, - symptoms=symptoms, - ) + with sim.population.individual_properties(person_id) as individual_properties: + sim.modules["Diarrhoea"].do_at_generic_first_appt( + person_id=person_id, + individual_properties=individual_properties, + schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event, + diagnosis_function=diagnosis_fn, + symptoms=symptoms, + ) evs = sim.modules['HealthSystem'].find_events_for_person(person_id) assert 1 == len(evs) assert isinstance(evs[0][1], HSI_Diarrhoea_Treatment_Outpatient) @@ -495,7 +498,6 @@ def test_do_when_presentation_with_diarrhoea_severe_dehydration_dxtest_notfuncti df.loc[person_id, props_new.keys()] = props_new.values() generic_hsi = HSI_GenericNonEmergencyFirstAppt( module=sim.modules['HealthSeekingBehaviour'], person_id=person_id) - patient_details = sim.population.row_in_readonly_form(person_id) symptoms = {"diarrhoea"} def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False): @@ -509,12 +511,14 @@ def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False): # Only an out-patient appointment should be created as the DxTest for danger signs is not functional. sim.modules['Diarrhoea'].parameters['prob_hospitalization_on_danger_signs'] = 0.0 sim.modules['HealthSystem'].reset_queue() - sim.modules["Diarrhoea"].do_at_generic_first_appt( - patient_id=person_id, - patient_details=patient_details, - diagnosis_function=diagnosis_fn, - symptoms=symptoms, - ) + with sim.population.individual_properties(person_id) as individual_properties: + sim.modules["Diarrhoea"].do_at_generic_first_appt( + person_id=person_id, + individual_properties=individual_properties, + schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event, + diagnosis_function=diagnosis_fn, + symptoms=symptoms, + ) evs = sim.modules['HealthSystem'].find_events_for_person(person_id) assert 1 == len(evs) assert isinstance(evs[0][1], HSI_Diarrhoea_Treatment_Outpatient) @@ -573,7 +577,6 @@ def test_do_when_presentation_with_diarrhoea_non_severe_dehydration(seed): df.loc[person_id, props_new.keys()] = props_new.values() generic_hsi = HSI_GenericNonEmergencyFirstAppt( module=sim.modules['HealthSeekingBehaviour'], person_id=person_id) - patient_details = sim.population.row_in_readonly_form(person_id) symptoms = {"diarrhoea"} def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False): @@ -585,12 +588,14 @@ def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False): ) # 1) Outpatient HSI should be created sim.modules["HealthSystem"].reset_queue() - sim.modules["Diarrhoea"].do_at_generic_first_appt( - patient_id=person_id, - patient_details=patient_details, - diagnosis_function=diagnosis_fn, - symptoms=symptoms, - ) + with sim.population.individual_properties(person_id) as individual_properties: + sim.modules["Diarrhoea"].do_at_generic_first_appt( + person_id=person_id, + individual_properties=individual_properties, + schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event, + diagnosis_function=diagnosis_fn, + symptoms=symptoms, + ) evs = sim.modules["HealthSystem"].find_events_for_person(person_id) assert 1 == len(evs) diff --git a/tests/test_malaria.py b/tests/test_malaria.py index 4ac2d377db..6fb185c433 100644 --- a/tests/test_malaria.py +++ b/tests/test_malaria.py @@ -281,7 +281,7 @@ def diagnosis_function(tests, use_dict: bool = False, report_tried: bool = False assert sim.modules['Malaria'].check_if_fever_is_caused_by_malaria( true_malaria_infection_type = df.at[person_id, "ma_inf_type"], diagnosis_function = diagnosis_function, - patient_id=person_id, + person_id=person_id, ) == expected_diagnosis @@ -362,7 +362,7 @@ def diagnosis_function(tests, use_dict: bool = False, report_tried: bool = False person_id, "ma_inf_type" ], diagnosis_function=diagnosis_function, - patient_id=person_id, + person_id=person_id, ) == "negative_malaria_test" ) diff --git a/tests/test_module_dependencies.py b/tests/test_module_dependencies.py index fd61bb40be..ca5bf58482 100644 --- a/tests/test_module_dependencies.py +++ b/tests/test_module_dependencies.py @@ -30,7 +30,11 @@ module_class_map = get_module_class_map( - excluded_modules={'Module', 'Skeleton', 'SimplifiedPregnancyAndLabour'} + excluded_modules={ + "Module", + "Skeleton", + "SimplifiedPregnancyAndLabour", + } ) diff --git a/tests/test_population.py b/tests/test_population.py new file mode 100644 index 0000000000..e8a549209d --- /dev/null +++ b/tests/test_population.py @@ -0,0 +1,216 @@ +import numpy as np +import pandas as pd +import pytest + +from tlo.core import Property, Types +from tlo.population import Population + + +@pytest.fixture +def properties(): + return { + f"{type_.name.lower()}_{i}": Property(type_, f"Column {i} of type {type_}") + for type_ in [Types.INT, Types.BOOL, Types.REAL, Types.DATE, Types.BITSET] + for i in range(5) + } + + +@pytest.fixture(params=[1, 100, 1000]) +def initial_size(request): + return request.param + + +@pytest.fixture(params=[None, 0.02, 0.1]) +def append_size(request, initial_size): + return ( + request.param + if request.param is None + else max(int(initial_size * request.param), 1) + ) + + +@pytest.fixture +def population(properties, initial_size, append_size): + return Population(properties, initial_size, append_size) + + +@pytest.fixture +def rng(seed): + return np.random.RandomState(seed % 2**32) + + +def _generate_random_values(property, rng, size=None): + if property.type_ == Types.DATE: + return np.datetime64("2010-01-01") + rng.randint(0, 4000, size=size) + elif property.type_ in (Types.INT, Types.BITSET): + return rng.randint(low=0, high=100, size=size) + elif property.type_ == Types.REAL: + return rng.standard_normal(size=size) + elif property.type_ == Types.BOOL: + return rng.uniform(size=size) < 0.5 + else: + msg = f"Unhandled type {property.type_}" + raise ValueError(msg) + + +@pytest.fixture +def population_with_random_property_values(population, properties, initial_size, rng): + + for name, property in properties.items(): + population.props[name] = pd.Series( + _generate_random_values(property, rng, initial_size), + dtype=property.pandas_type, + ) + + return population + + +def test_population_invalid_append_size_raises(properties, initial_size): + with pytest.raises(AssertionError, match="greater than 0"): + Population(properties, initial_size, append_size=-1) + + +def test_population_attributes(population, properties, initial_size, append_size): + assert population.initial_size == initial_size + assert population.next_person_id == initial_size + if append_size is not None: + assert len(population.new_rows) == append_size + else: + assert 0 < len(population.new_rows) <= initial_size + assert len(population.props.index) == initial_size + assert len(population.props.columns) == len(properties) + assert set(population.props.columns) == properties.keys() + assert all( + properties[name].pandas_type == col.dtype + for name, col in population.props.items() + ) + + +def test_population_do_birth(population): + initial_population_props_copy = population.props.copy() + initial_size = population.initial_size + append_size = len(population.new_rows) + + def check_population(population, birth_number): + expected_next_person_id = initial_size + birth_number + # population size should increase by append_size on first birth and after + # every subsequent append_size births by a further append_size + expected_size = ( + initial_size + ((birth_number - 1) // append_size + 1) * append_size + ) + assert all(initial_population_props_copy.columns == population.props.columns) + assert all(initial_population_props_copy.dtypes == population.props.dtypes) + assert population.next_person_id == expected_next_person_id + assert len(population.props.index) == expected_size + + for birth_number in range(1, append_size + 2): + population.do_birth() + check_population(population, birth_number) + + +def test_population_individual_properties_read_only_write_raises( + population, properties +): + with population.individual_properties( + person_id=0, read_only=True + ) as individual_properties: + for property_name in properties: + with pytest.raises(ValueError, match="read-only"): + individual_properties[property_name] = 0 + + +@pytest.mark.parametrize("read_only", [True, False]) +@pytest.mark.parametrize("person_id", [0, 1, -1]) +def test_population_individual_properties_read( + population_with_random_property_values, properties, rng, read_only, person_id +): + person_id = person_id % population_with_random_property_values.initial_size + population_dataframe = population_with_random_property_values.props + with population_with_random_property_values.individual_properties( + person_id=person_id, read_only=read_only + ) as individual_properties: + for property_name in properties: + assert ( + individual_properties[property_name] + == population_dataframe.at[person_id, property_name] + ) + # Try reading all properties (in a new random order) a second time to check any + # caching mechanism is working as expected + shuffled_property_names = list(properties.keys()) + rng.shuffle(shuffled_property_names) + for property_name in shuffled_property_names: + assert ( + individual_properties[property_name] + == population_dataframe.at[person_id, property_name] + ) + + +@pytest.mark.parametrize("read_only", [True, False]) +@pytest.mark.parametrize("person_id", [0, 1, -1]) +def test_population_individual_properties_access_raises_when_finalized( + population_with_random_property_values, properties, rng, read_only, person_id +): + person_id = person_id % population_with_random_property_values.initial_size + with population_with_random_property_values.individual_properties( + person_id=person_id, read_only=read_only + ) as individual_properties: + pass + for property_name in properties: + with pytest.raises(ValueError, match="finalized"): + individual_properties[property_name] + with pytest.raises(ValueError, match="finalized"): + individual_properties[property_name] = None + + +@pytest.mark.parametrize("person_id", [0, 1, -1]) +def test_population_individual_properties_write_with_context_manager( + population_with_random_property_values, properties, rng, person_id +): + initial_population_dataframe = population_with_random_property_values.props.copy() + person_id = person_id % population_with_random_property_values.initial_size + updated_values = {} + with population_with_random_property_values.individual_properties( + person_id=person_id, read_only=False + ) as individual_properties: + for property_name, property in properties.items(): + updated_values[property_name] = _generate_random_values(property, rng) + individual_properties[property_name] = updated_values[property_name] + # Population dataframe should see updated properties for person_id row + for property_name, property in properties.items(): + assert ( + population_with_random_property_values.props.at[person_id, property_name] + == updated_values[property_name] + ) + # All other rows in population dataframe should be unchanged + all_rows_except_updated = ~initial_population_dataframe.index.isin([person_id]) + assert population_with_random_property_values.props[all_rows_except_updated].equals( + initial_population_dataframe[all_rows_except_updated] + ) + + +@pytest.mark.parametrize("person_id", [0, 1, -1]) +def test_population_individual_properties_write_with_sync( + population_with_random_property_values, properties, rng, person_id +): + initial_population_dataframe = population_with_random_property_values.props.copy() + person_id = person_id % population_with_random_property_values.initial_size + updated_values = {} + with population_with_random_property_values.individual_properties( + person_id=person_id, read_only=False + ) as individual_properties: + for property_name, property in properties.items(): + updated_values[property_name] = _generate_random_values(property, rng) + individual_properties[property_name] = updated_values[property_name] + # Before synchronization all values in population dataframe should be unchanged + assert initial_population_dataframe.equals( + population_with_random_property_values.props + ) + individual_properties.synchronize_updates_to_dataframe() + # After synchronization all values in population dataframe should be updated + for property_name, property in properties.items(): + assert ( + population_with_random_property_values.props.at[ + person_id, property_name + ] + == updated_values[property_name] + ) diff --git a/tests/test_stunting.py b/tests/test_stunting.py index 4b11db1512..f41bab7c78 100644 --- a/tests/test_stunting.py +++ b/tests/test_stunting.py @@ -235,15 +235,17 @@ def test_routine_assessment_for_chronic_undernutrition_if_stunted_and_correctly_ person_id = 0 df.loc[person_id, 'age_years'] = 2 df.loc[person_id, "un_HAZ_category"] = "-3<=HAZ<-2" - patient_details = sim.population.row_in_readonly_form(person_id) # Make the probability of stunting checking/diagnosis as 1.0 sim.modules['Stunting'].parameters['prob_stunting_diagnosed_at_generic_appt'] = 1.0 - # Subject the person to `do_at_generic_first_appt` - sim.modules["Stunting"].do_at_generic_first_appt( - patient_id=person_id, patient_details=patient_details - ) + with sim.population.individual_properties(person_id) as individual_properties: + # Subject the person to `do_at_generic_first_appt` + sim.modules["Stunting"].do_at_generic_first_appt( + person_id=person_id, + individual_properties=individual_properties, + schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event, + ) # Check that there is an HSI scheduled for this person hsi_event_scheduled = [ @@ -302,13 +304,17 @@ def test_routine_assessment_for_chronic_undernutrition_if_stunted_but_no_checkin person_id = 0 df.loc[person_id, 'age_years'] = 2 df.loc[person_id, "un_HAZ_category"] = "HAZ<-3" - patient_details = sim.population.row_in_readonly_form(person_id) # Make the probability of stunting checking/diagnosis as 0.0 sim.modules['Stunting'].parameters['prob_stunting_diagnosed_at_generic_appt'] = 0.0 - # Subject the person to `do_at_generic_first_appt` - sim.modules["Stunting"].do_at_generic_first_appt(patient_id=person_id, patient_details=patient_details) + with sim.population.individual_properties(person_id) as individual_properties: + # Subject the person to `do_at_generic_first_appt` + sim.modules['Stunting'].do_at_generic_first_appt( + person_id=person_id, + individual_properties=individual_properties, + schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event, + ) # Check that there is no HSI scheduled for this person hsi_event_scheduled = [ @@ -320,10 +326,13 @@ def test_routine_assessment_for_chronic_undernutrition_if_stunted_but_no_checkin # Then make the probability of stunting checking/diagnosis 1.0 # and check the HSI is scheduled for this person - sim.modules["Stunting"].parameters["prob_stunting_diagnosed_at_generic_appt"] = 1.0 - sim.modules["Stunting"].do_at_generic_first_appt( - patient_id=person_id, patient_details=patient_details - ) + sim.modules['Stunting'].parameters["prob_stunting_diagnosed_at_generic_appt"] = 1.0 + with sim.population.individual_properties(person_id) as individual_properties: + sim.modules['Stunting'].do_at_generic_first_appt( + person_id=person_id, + individual_properties=individual_properties, + schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event, + ) hsi_event_scheduled = [ ev[1] for ev in sim.modules["HealthSystem"].find_events_for_person(person_id) @@ -346,10 +355,13 @@ def test_routine_assessment_for_chronic_undernutrition_if_not_stunted(seed): person_id = 0 df.loc[person_id, 'age_years'] = 2 df.loc[person_id, 'un_HAZ_category'] = 'HAZ>=-2' - patient_details = sim.population.row_in_readonly_form(person_id) - - # Subject the person to `do_at_generic_first_appt` - sim.modules["Stunting"].do_at_generic_first_appt(patient_id=person_id, patient_details=patient_details) + with sim.population.individual_properties(person_id) as individual_properties: + # Subject the person to `do_at_generic_first_appt` + sim.modules["Stunting"].do_at_generic_first_appt( + person_id=person_id, + individual_properties=individual_properties, + schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event, + ) # Check that there is no HSI scheduled for this person hsi_event_scheduled = [ From 802c3641603c5e53888b846c79ba87d89655aa2b Mon Sep 17 00:00:00 2001 From: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> Date: Wed, 19 Jun 2024 17:43:18 +0100 Subject: [PATCH 02/19] Create function get_parameters_for_standard_mode2_runs (#1398) --- src/tlo/analysis/utils.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/tlo/analysis/utils.py b/src/tlo/analysis/utils.py index 344a094ef3..201f2fb25e 100644 --- a/src/tlo/analysis/utils.py +++ b/src/tlo/analysis/utils.py @@ -1129,6 +1129,41 @@ def get_parameters_for_status_quo() -> Dict: "equip_availability": "all", # <--- NB. Existing calibration is assuming all equipment is available }, } + +def get_parameters_for_standard_mode2_runs() -> Dict: + """ + Returns a dictionary of parameters and their updated values to indicate + the "standard mode 2" scenario. + + The return dict is in the form: + e.g. { + 'Depression': { + 'pr_assessed_for_depression_for_perinatal_female': 1.0, + 'pr_assessed_for_depression_in_generic_appt_level1': 1.0, + }, + 'Hiv': { + 'prob_start_art_or_vs': 1.0, + } + } + """ + + return { + "SymptomManager": { + "spurious_symptoms": True, + }, + "HealthSystem": { + 'Service_Availability': ['*'], + "use_funded_or_actual_staffing": "actual", + "mode_appt_constraints": 1, + "mode_appt_constraints_postSwitch": 2, # <-- Include a transition to mode 2, to pick up any issues with this + "year_mode_switch": 2012, # <-- Could make this quite soon, but I'd say >1 year + "tclose_overwrite": 1, # <-- In most of our runs in mode 2, we chose to overwrite tclose + "tclose_days_offset_overwrite": 7, # <-- and usually set it to 7. + "cons_availability": "default", + "beds_availability": "default", + "equip_availability": "all", # <--- NB. Existing calibration is assuming all equipment is available + }, + } def get_parameters_for_improved_healthsystem_and_healthcare_seeking( From f5251b65efe6a8a058cf7d9408c55d3354f38909 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Thu, 20 Jun 2024 17:51:12 +0100 Subject: [PATCH 03/19] Equipment: Integration into modules (#1341) --- .../ResourceFile_EquipmentCatalogue.csv | 4 +- ...eFile_Equipment_Availability_Estimates.csv | 4 +- .../equipment_availability_estimation.py | 6 ++ src/tlo/methods/alri.py | 19 ++++- src/tlo/methods/bladder_cancer.py | 24 ++++--- src/tlo/methods/breast_cancer.py | 15 ++-- src/tlo/methods/cancer_consumables.py | 24 ++----- src/tlo/methods/cardio_metabolic_disorders.py | 19 +++++ .../methods/care_of_women_during_pregnancy.py | 69 +++++++++++++++---- src/tlo/methods/contraception.py | 20 +++++- src/tlo/methods/copd.py | 1 + src/tlo/methods/diarrhoea.py | 3 +- src/tlo/methods/equipment.py | 49 ++++++++++--- src/tlo/methods/hiv.py | 4 ++ src/tlo/methods/labour.py | 37 +++++++--- src/tlo/methods/malaria.py | 4 ++ src/tlo/methods/measles.py | 3 + src/tlo/methods/newborn_outcomes.py | 2 + src/tlo/methods/oesophagealcancer.py | 16 +++-- src/tlo/methods/other_adult_cancers.py | 12 ++-- src/tlo/methods/postnatal_supervisor.py | 2 + src/tlo/methods/prostate_cancer.py | 12 ++-- src/tlo/methods/rti.py | 25 ++++++- src/tlo/methods/schisto.py | 2 + src/tlo/methods/symptommanager.py | 3 +- src/tlo/methods/tb.py | 19 +++++ tests/test_equipment.py | 36 +++++++--- 27 files changed, 330 insertions(+), 104 deletions(-) diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv index 33ba052c64..45f801f0c8 100644 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3e151e16f7eea2ae61d2fa637c26449aa533ddc6a7f0d83aff495f5f6c9d1f8d -size 33201 +oid sha256:ec5f619816df6150ae92839152607296a5f2289024c92ce6b5ba621d38db20b7 +size 33517 diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment_Availability_Estimates.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment_Availability_Estimates.csv index 3f0739577a..706297da67 100644 --- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment_Availability_Estimates.csv +++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment_Availability_Estimates.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e31936377f6b90779ad66480c4bc477cdca9322e86f2e00d202bbb91eebf6d57 -size 1306170 +oid sha256:2785365d20a4da4c147ba6a5df9e0259c9076df0fec556086aea0f2a068c9c53 +size 1313098 diff --git a/src/scripts/data_file_processing/healthsystem/equipment/equipment_availability_estimation.py b/src/scripts/data_file_processing/healthsystem/equipment/equipment_availability_estimation.py index 0a742d37b8..12ba3c7f9d 100644 --- a/src/scripts/data_file_processing/healthsystem/equipment/equipment_availability_estimation.py +++ b/src/scripts/data_file_processing/healthsystem/equipment/equipment_availability_estimation.py @@ -322,6 +322,12 @@ .drop_duplicates() \ .pipe(lambda x: x.set_index(x['Item_code'].astype(int)))['Category'] \ .to_dict() +# Manually declare the price category for equipment items added manually +# 402: Endoscope: 'Cost >= $1000' +equipment_price_category_mapper[402] = 'Cost >= $1000' +# 403: Electrocardiogram: 'Cost >= $1000' +equipment_price_category_mapper[403] = 'Cost >= $1000' + equipment_price_category = final_equipment_availability_export_full.index.get_level_values('Item_Code') \ .map(equipment_price_category_mapper) final_equipment_availability_export_full = final_equipment_availability_export_full.groupby( diff --git a/src/tlo/methods/alri.py b/src/tlo/methods/alri.py index 8c1ab41401..c27a54dd30 100644 --- a/src/tlo/methods/alri.py +++ b/src/tlo/methods/alri.py @@ -2551,6 +2551,8 @@ def _get_disease_classification_for_treatment_decision(self, 'chest_indrawing_pneumonia', (symptoms-based assessment) 'cough_or_cold' (symptoms-based assessment) }.""" + if use_oximeter: + self.add_equipment({'Pulse oximeter'}) child_is_younger_than_2_months = age_exact_years < (2.0 / 12.0) @@ -2606,6 +2608,15 @@ def _try_treatment(antibiotic_indicated: Tuple[str], oxygen_indicated: bool) -> oxygen_available = self._get_cons('Oxygen_Therapy') oxygen_provided = (oxygen_available and oxygen_indicated) + # If individual is provided with oxygen, add used equipment + if oxygen_provided: + self.add_equipment({'Oxygen cylinder, with regulator', 'Nasal Prongs'}) + + # If individual is provided with intravenous antibiotics, add used equipment + if antibiotic_provided in ('1st_line_IV_antibiotics', + 'Benzylpenicillin_gentamicin_therapy_for_severe_pneumonia'): + self.add_equipment({'Infusion pump', 'Drip stand'}) + all_things_needed_available = antibiotic_available and ( (oxygen_available and oxygen_indicated) or (not oxygen_indicated) ) @@ -2687,6 +2698,7 @@ def _provide_bronchodilator_if_wheeze(self, facility_level, symptoms): if facility_level == '1a': _ = self._get_cons('Inhaled_Brochodilator') else: + # n.b. this is never called, see issue 1172 _ = self._get_cons('Brochodilator_and_Steroids') def do_on_follow_up_following_treatment_failure(self): @@ -2694,9 +2706,12 @@ def do_on_follow_up_following_treatment_failure(self): A further drug will be used but this will have no effect on the chance of the person dying.""" if self._has_staph_aureus(): - _ = self._get_cons('2nd_line_Antibiotic_therapy_for_severe_staph_pneumonia') + cons_avail = self._get_cons('2nd_line_Antibiotic_therapy_for_severe_staph_pneumonia') else: - _ = self._get_cons('Ceftriaxone_therapy_for_severe_pneumonia') + cons_avail = self._get_cons('Ceftriaxone_therapy_for_severe_pneumonia') + + if cons_avail: + self.add_equipment({'Infusion pump', 'Drip stand'}) def apply(self, person_id, squeeze_factor): """Assess and attempt to treat the person.""" diff --git a/src/tlo/methods/bladder_cancer.py b/src/tlo/methods/bladder_cancer.py index 78899d4705..113d19fde2 100644 --- a/src/tlo/methods/bladder_cancer.py +++ b/src/tlo/methods/bladder_cancer.py @@ -725,14 +725,14 @@ def apply(self, person_id, squeeze_factor): return hs.get_blank_appt_footprint() # Check consumables are available - # TODO: replace with cystoscope - cons_avail = self.get_consumables(item_codes=self.module.item_codes_bladder_can['screening_biopsy_core'], - optional_item_codes= - self.module.item_codes_bladder_can['screening_biopsy_optional']) + cons_avail = self.get_consumables(item_codes=self.module.item_codes_bladder_can['screening_cystoscopy_core'], + optional_item_codes=self.module.item_codes_bladder_can[ + 'screening_biopsy_endoscopy_cystoscopy_optional']) if cons_avail: # Use a biopsy to diagnose whether the person has bladder Cancer - # If consumables are available, run the dx_test representing the biopsy + # If consumables are available update the use of equipment and run the dx_test representing the biopsy + self.add_equipment({'Cystoscope', 'Ordinary Microscope', 'Ultrasound scanning machine'}) # Use a cystoscope to diagnose whether the person has bladder Cancer: dx_result = hs.dx_manager.run_dx_test( @@ -798,14 +798,14 @@ def apply(self, person_id, squeeze_factor): return hs.get_blank_appt_footprint() # Check consumables are available - # TODO: replace with cystoscope - cons_avail = self.get_consumables(item_codes=self.module.item_codes_bladder_can['screening_biopsy_core'], + cons_avail = self.get_consumables(item_codes=self.module.item_codes_bladder_can['screening_cystoscopy_core'], optional_item_codes=self.module.item_codes_bladder_can[ - 'screening_biopsy_optional']) + 'screening_biopsy_endoscopy_cystoscopy_optional']) if cons_avail: # Use a biopsy to diagnose whether the person has bladder Cancer - # If consumables are available, run the dx_test representing the biopsy + # If consumables are available log the use of equipment and run the dx_test representing the biopsy + self.add_equipment({'Cystoscope', 'Ordinary Microscope', 'Ultrasound scanning machine'}) # Use a cystoscope to diagnose whether the person has bladder Cancer: dx_result = hs.dx_manager.run_dx_test( @@ -894,7 +894,8 @@ def apply(self, person_id, squeeze_factor): self.module.item_codes_bladder_can['treatment_surgery_optional']) if cons_avail: - # If consumables are available and the treatment will go ahead + # If consumables are available and the treatment will go ahead - update the equipment + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery')) # Record date and stage of starting treatment df.at[person_id, "bc_date_treatment"] = self.sim.date @@ -998,7 +999,8 @@ def apply(self, person_id, squeeze_factor): item_codes=self.module.item_codes_bladder_can['palliation']) if cons_available: - # If consumables are available and the treatment will go ahead + # If consumables are available and the treatment will go ahead - update the equipment + self.add_equipment({'Infusion pump', 'Drip stand'}) # Record the start of palliative care if this is first appointment if pd.isnull(df.at[person_id, "bc_date_palliative_care"]): diff --git a/src/tlo/methods/breast_cancer.py b/src/tlo/methods/breast_cancer.py index 9ef5dc3e41..d362f7ce08 100644 --- a/src/tlo/methods/breast_cancer.py +++ b/src/tlo/methods/breast_cancer.py @@ -696,11 +696,13 @@ def apply(self, person_id, squeeze_factor): # Check consumables to undertake biopsy are available cons_avail = self.get_consumables(item_codes=self.module.item_codes_breast_can['screening_biopsy_core'], optional_item_codes= - self.module.item_codes_breast_can['screening_biopsy_optional']) + self.module.item_codes_breast_can[ + 'screening_biopsy_endoscopy_cystoscopy_optional']) if cons_avail: # Use a biopsy to diagnose whether the person has breast Cancer - # If consumables are available, run the dx_test representing the biopsy + # If consumables are available, add the used equipment and run the dx_test representing the biopsy + self.add_equipment({'Ultrasound scanning machine', 'Ordinary Microscope'}) dx_result = hs.dx_manager.run_dx_test( dx_tests_to_run='biopsy_for_breast_cancer_given_breast_lump_discernible', @@ -764,8 +766,6 @@ def apply(self, person_id, squeeze_factor): df = self.sim.population.props hs = self.sim.modules["HealthSystem"] - # todo: request consumables needed for this - if not df.at[person_id, 'is_alive']: return hs.get_blank_appt_footprint() @@ -798,7 +798,9 @@ def apply(self, person_id, squeeze_factor): ) if cons_available: - # If consumables, treatment will go ahead + # If consumables are available and the treatment will go ahead - add the used equipment + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery')) + # Log the use of adjuvant chemotherapy self.get_consumables( item_codes=self.module.item_codes_breast_can['treatment_chemotherapy'], @@ -906,7 +908,8 @@ def apply(self, person_id, squeeze_factor): item_codes=self.module.item_codes_breast_can['palliation']) if cons_available: - # If consumables are available and the treatment will go ahead + # If consumables are available and the treatment will go ahead - add the used equipment + self.add_equipment({'Infusion pump', 'Drip stand'}) # Record the start of palliative care if this is first appointment if pd.isnull(df.at[person_id, "brc_date_palliative_care"]): diff --git a/src/tlo/methods/cancer_consumables.py b/src/tlo/methods/cancer_consumables.py index 2649626b2f..e26d577242 100644 --- a/src/tlo/methods/cancer_consumables.py +++ b/src/tlo/methods/cancer_consumables.py @@ -15,16 +15,16 @@ def get_consumable_item_codes_cancers(self) -> Dict[str, int]: cons_dict = dict() # Add items that are needed for all cancer modules - cons_dict['screening_biopsy_core'] = \ - {get_item_code("Biopsy needle"): 1} - - cons_dict['screening_biopsy_optional'] = \ + cons_dict['screening_biopsy_endoscopy_cystoscopy_optional'] = \ {get_item_code("Specimen container"): 1, get_item_code("Lidocaine HCl (in dextrose 7.5%), ampoule 2 ml"): 1, get_item_code("Gauze, absorbent 90cm x 40m_each_CMST"): 30, get_item_code("Disposables gloves, powder free, 100 pieces per box"): 1, get_item_code("Syringe, needle + swab"): 1} + cons_dict['screening_biopsy_core'] = \ + {get_item_code("Biopsy needle"): 1} + cons_dict['treatment_surgery_core'] = \ {get_item_code("Halothane (fluothane)_250ml_CMST"): 100, get_item_code("Scalpel blade size 22 (individually wrapped)_100_CMST"): 1} @@ -69,23 +69,9 @@ def get_consumable_item_codes_cancers(self) -> Dict[str, int]: cons_dict['screening_cystoscopy_core'] = \ {get_item_code("Cystoscope"): 1} - cons_dict['screening_cystoscope_optional'] = \ - {get_item_code("Specimen container"): 1, - get_item_code("Lidocaine HCl (in dextrose 7.5%), ampoule 2 ml"): 1, - get_item_code("Gauze, absorbent 90cm x 40m_each_CMST"): 30, - get_item_code("Disposables gloves, powder free, 100 pieces per box"): 1, - get_item_code("Syringe, needle + swab"): 1} - elif 'OesophagealCancer' == self.name: - cons_dict['screening_endoscope_core'] = \ + cons_dict['screening_endoscopy_core'] = \ {get_item_code("Endoscope"): 1} - cons_dict['screening_endoscope_optional'] = \ - {get_item_code("Specimen container"): 1, - get_item_code("Gauze, absorbent 90cm x 40m_each_CMST"): 30, - get_item_code("Lidocaine HCl (in dextrose 7.5%), ampoule 2 ml"): 1, - get_item_code("Disposables gloves, powder free, 100 pieces per box"): 1, - get_item_code("Syringe, needle + swab"): 1} - return cons_dict diff --git a/src/tlo/methods/cardio_metabolic_disorders.py b/src/tlo/methods/cardio_metabolic_disorders.py index c46ea7b37e..d90688adb0 100644 --- a/src/tlo/methods/cardio_metabolic_disorders.py +++ b/src/tlo/methods/cardio_metabolic_disorders.py @@ -1435,6 +1435,7 @@ def apply(self, person_id, squeeze_factor): return hs.get_blank_appt_footprint() # Run a test to diagnose whether the person has condition: + self.add_equipment({'Blood pressure machine'}) dx_result = hs.dx_manager.run_dx_test( dx_tests_to_run='assess_hypertension', hsi_event=self @@ -1487,6 +1488,9 @@ def do_for_each_condition(self, _c) -> bool: if df.at[person_id, f'nc_{_c}_ever_diagnosed']: return + if _c == 'chronic_ischemic_heart_disease': + self.add_equipment({'Electrocardiogram', 'Stethoscope'}) + # Run a test to diagnose whether the person has condition: dx_result = hs.dx_manager.run_dx_test( dx_tests_to_run=f'assess_{_c}', @@ -1519,6 +1523,11 @@ def apply(self, person_id, squeeze_factor): return hs.get_blank_appt_footprint() # Do test and trigger treatment (if necessary) for each condition: + if set(self.conditions_to_investigate).intersection( + ['diabetes', 'chronic_kidney_disease', 'chronic_ischemic_hd'] + ): + self.add_equipment({'Analyser, Haematology', 'Analyser, Combined Chemistry and Electrolytes'}) + hsi_scheduled = [self.do_for_each_condition(_c) for _c in self.conditions_to_investigate] # If no follow-up treatment scheduled but the person has at least 2 risk factors, start weight loss treatment @@ -1542,6 +1551,7 @@ def apply(self, person_id, squeeze_factor): and (self.module.rng.rand() < self.module.parameters['hypertension_hsi']['pr_assessed_other_symptoms']) ): # Run a test to diagnose whether the person has condition: + self.add_equipment({'Blood pressure machine'}) dx_result = hs.dx_manager.run_dx_test( dx_tests_to_run='assess_hypertension', hsi_event=self @@ -1589,6 +1599,8 @@ def apply(self, person_id, squeeze_factor): # Don't advise those with CKD to lose weight, but do so for all other conditions if BMI is higher than normal if self.condition != 'chronic_kidney_disease' and (df.at[person_id, 'li_bmi'] > 2): + self.add_equipment({'Weighing scale'}) + self.sim.population.props.at[person_id, 'nc_ever_weight_loss_treatment'] = True # Schedule a post-weight loss event for individual to potentially lose weight in next 6-12 months: self.sim.schedule_event(CardioMetabolicDisordersWeightLossEvent(m, person_id), @@ -1749,6 +1761,12 @@ def do_for_each_event_to_be_investigated(self, _ev): df = self.sim.population.props # Run a test to diagnose whether the person has condition: + if _ev == 'ever_stroke': + self.add_equipment({'Computed Tomography (CT machine)', 'CT scanner accessories'}) + + if _ev == 'ever_heart_attack': + self.add_equipment({'Electrocardiogram'}) + dx_result = self.sim.modules['HealthSystem'].dx_manager.run_dx_test( dx_tests_to_run=f'assess_{_ev}', hsi_event=self @@ -1808,6 +1826,7 @@ def apply(self, person_id, squeeze_factor): data=('This is HSI_CardioMetabolicDisorders_SeeksEmergencyCareAndGetsTreatment: ' f'The squeeze-factor is {squeeze_factor}.'), ) + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ICU')) for _ev in self.events_to_investigate: self.do_for_each_event_to_be_investigated(_ev) diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py index 4025941254..dba3bcda8e 100644 --- a/src/tlo/methods/care_of_women_during_pregnancy.py +++ b/src/tlo/methods/care_of_women_during_pregnancy.py @@ -747,6 +747,7 @@ def screening_interventions_delivered_at_every_contact(self, hsi_event): # The process is repeated for blood pressure monitoring if self.rng.random_sample() < params['prob_intervention_delivered_bp']: + hsi_event.add_equipment({'Sphygmomanometer'}) if self.sim.modules['HealthSystem'].dx_manager.run_dx_test(dx_tests_to_run='blood_pressure_measurement', hsi_event=hsi_event): @@ -1081,20 +1082,24 @@ def gdm_screening(self, hsi_event): # If the test accurately detects a woman has gestational diabetes the consumables are recorded and # she is referred for treatment - if avail and self.sim.modules['HealthSystem'].dx_manager.run_dx_test( - dx_tests_to_run='blood_test_glucose', hsi_event=hsi_event): + if avail: + hsi_event.add_equipment({'Glucometer'}) - logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'gdm_screen'}) - mni[person_id]['anc_ints'].append('gdm_screen') + if ( + self.sim.modules['HealthSystem'].dx_manager.run_dx_test( + dx_tests_to_run='blood_test_glucose', hsi_event=hsi_event) + ): + logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'gdm_screen'}) + mni[person_id]['anc_ints'].append('gdm_screen') - # We assume women with a positive GDM screen will be admitted (if they are not already receiving - # outpatient care) - if df.at[person_id, 'ac_gest_diab_on_treatment'] == 'none': + # We assume women with a positive GDM screen will be admitted (if they are not already receiving + # outpatient care) + if df.at[person_id, 'ac_gest_diab_on_treatment'] == 'none': - # Store onset after diagnosis as daly weight is tied to diagnosis - pregnancy_helper_functions.store_dalys_in_mni(person_id, mni, 'gest_diab_onset', - self.sim.date) - df.at[person_id, 'ac_to_be_admitted'] = True + # Store onset after diagnosis as daly weight is tied to diagnosis + pregnancy_helper_functions.store_dalys_in_mni(person_id, mni, 'gest_diab_onset', + self.sim.date) + df.at[person_id, 'ac_to_be_admitted'] = True def interventions_delivered_each_visit_from_anc2(self, hsi_event): """This function contains a collection of interventions that are delivered to women every time they attend ANC @@ -1213,6 +1218,7 @@ def full_blood_count_testing(self, hsi_event): # If a woman is not truly anaemic but the FBC returns a result of anaemia, due to tests specificity, we # assume the reported anaemia is mild hsi_event.get_consumables(item_codes=self.item_codes_preg_consumables['blood_test_equipment']) + hsi_event.add_equipment({'Analyser, Haematology'}) test_result = self.sim.modules['HealthSystem'].dx_manager.run_dx_test( dx_tests_to_run='full_blood_count_hb', hsi_event=hsi_event) @@ -1255,6 +1261,8 @@ def antenatal_blood_transfusion(self, individual_id, hsi_event): if avail and sf_check: pregnancy_helper_functions.log_met_need(self, 'blood_tran', hsi_event) + hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) + # If the woman is receiving blood due to anaemia we apply a probability that a transfusion of 2 units # RBCs will correct this woman's severe anaemia if params['treatment_effect_blood_transfusion_anaemia'] > self.rng.random_sample(): @@ -1303,6 +1311,7 @@ def initiate_treatment_for_severe_hypertension(self, individual_id, hsi_event): # If they are available then the woman is started on treatment if avail: pregnancy_helper_functions.log_met_need(self, 'iv_htns', hsi_event) + hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) # We assume women treated with antihypertensives would no longer be severely hypertensive- meaning they # are not at risk of death from severe gestational hypertension in the PregnancySupervisor event @@ -1341,6 +1350,7 @@ def treatment_for_severe_pre_eclampsia_or_eclampsia(self, individual_id, hsi_eve if avail and sf_check: df.at[individual_id, 'ac_mag_sulph_treatment'] = True pregnancy_helper_functions.log_met_need(self, 'mag_sulph', hsi_event) + hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) def antibiotics_for_prom(self, individual_id, hsi_event): """ @@ -1363,6 +1373,7 @@ def antibiotics_for_prom(self, individual_id, hsi_event): if avail and sf_check: df.at[individual_id, 'ac_received_abx_for_prom'] = True + hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) def ectopic_pregnancy_treatment_doesnt_run(self, hsi_event): """ @@ -1452,6 +1463,12 @@ def apply(self, person_id, squeeze_factor): df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1 # =================================== INTERVENTIONS ==================================================== + # Add equipment used during first ANC visit not directly related to interventions + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC')) + self.add_equipment( + {'Height Pole (Stadiometer)', 'MUAC tape', + 'Ultrasound, combined 2/4 pole interferential with vacuum and dual frequency 1-3MHZ'}) + # First all women, regardless of ANC contact or gestation, undergo urine and blood pressure measurement # and depression screening self.module.screening_interventions_delivered_at_every_contact(hsi_event=self) @@ -1470,6 +1487,7 @@ def apply(self, person_id, squeeze_factor): # If the woman presents after 20 weeks she is provided interventions she has missed by presenting late if mother.ps_gestational_age_in_weeks > 19: + self.add_equipment({'Stethoscope, foetal, monaural, Pinard, plastic'}) self.module.point_of_care_hb_testing(hsi_event=self) self.module.albendazole_administration(hsi_event=self) self.module.iptp_administration(hsi_event=self) @@ -1534,7 +1552,12 @@ def apply(self, person_id, squeeze_factor): df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1 # =================================== INTERVENTIONS ==================================================== - # First we administer the administer the interventions all women will receive at this contact regardless of + # Add equipment used during ANC visit not directly related to interventions + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC')) + self.add_equipment( + {'Ultrasound, combined 2/4 pole interferential with vacuum and dual frequency 1-3MHZ'}) + + # First we administer the interventions all women will receive at this contact regardless of # gestational age self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self) self.module.tetanus_vaccination(hsi_event=self) @@ -1618,6 +1641,8 @@ def apply(self, person_id, squeeze_factor): df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1 # =================================== INTERVENTIONS ==================================================== + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC')) + gest_age_next_contact = self.module.determine_gestational_age_for_next_contact(person_id) self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self) @@ -1690,6 +1715,8 @@ def apply(self, person_id, squeeze_factor): df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1 # =================================== INTERVENTIONS ==================================================== + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC')) + gest_age_next_contact = self.module.determine_gestational_age_for_next_contact(person_id) self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self) @@ -1757,7 +1784,11 @@ def apply(self, person_id, squeeze_factor): self.module.anc_counter[5] += 1 df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1 - # =================================== INTERVENTIONS ==================================================== + # =================================== INTERVENTIONS =================================================== + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC')) + self.add_equipment( + {'Ultrasound, combined 2/4 pole interferential with vacuum and dual frequency 1-3MHZ'}) + gest_age_next_contact = self.module.determine_gestational_age_for_next_contact(person_id) self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self) @@ -1825,6 +1856,9 @@ def apply(self, person_id, squeeze_factor): gest_age_next_contact = self.module.determine_gestational_age_for_next_contact(person_id) # =================================== INTERVENTIONS ==================================================== + self.add_equipment({'Weighing scale', 'Measuring tapes', + 'Stethoscope, foetal, monaural, Pinard, plastic'}) + self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self) if mother.ps_gestational_age_in_weeks < 40: @@ -1884,6 +1918,8 @@ def apply(self, person_id, squeeze_factor): df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1 # =================================== INTERVENTIONS ==================================================== + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC')) + gest_age_next_contact = self.module.determine_gestational_age_for_next_contact(person_id) self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self) @@ -1937,6 +1973,8 @@ def apply(self, person_id, squeeze_factor): self.module.anc_counter[8] += 1 df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1 + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC')) + self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self) if df.at[person_id, 'ac_to_be_admitted']: @@ -2559,6 +2597,10 @@ def apply(self, person_id, squeeze_factor): sf='retained_prod', hsi_event=self) + # Add used equipment if intervention can happen + if baseline_cons and sf_check: + self.add_equipment({'D&C set', 'Suction Curettage machine', 'Drip stand', 'Infusion pump'}) + # Then we determine if a woman gets treatment for her complication depending on availability of the baseline # consumables (misoprostol) or a HCW who can conduct MVA/DC (we dont model equipment) and additional # consumables for management of her specific complication @@ -2643,6 +2685,7 @@ def apply(self, person_id, squeeze_factor): if avail: self.sim.modules['PregnancySupervisor'].mother_and_newborn_info[person_id]['delete_mni'] = True pregnancy_helper_functions.log_met_need(self.module, 'ep_case_mang', self) + self.add_equipment({'Laparotomy Set'}) # For women who have sought care after they have experienced rupture we use this treatment variable to # reduce risk of death (women who present prior to rupture do not pass through the death event as we assume diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py index 580125f7ba..ab6c633f4c 100644 --- a/src/tlo/methods/contraception.py +++ b/src/tlo/methods/contraception.py @@ -1181,6 +1181,11 @@ def apply(self, person_id, squeeze_factor): # Record the date that Family Planning Appointment happened for this person self.sim.population.props.at[person_id, "co_date_of_last_fp_appt"] = self.sim.date + # Measure weight, height and BP even if contraception not administrated + self.add_equipment({ + 'Weighing scale', 'Height Pole (Stadiometer)', 'Blood pressure machine' + }) + # Determine essential and optional items items_essential = self.module.cons_codes[self.new_contraceptive] items_optional = {} @@ -1206,7 +1211,8 @@ def apply(self, person_id, squeeze_factor): items_all = {**items_essential, **items_optional} # Determine whether the contraception is administrated (ie all essential items are available), - # if so do log the availability of all items, if not set the contraception to "not_using": + # if so do log the availability of all items and update used equipment if any, if not set the contraception to + # "not_using": co_administrated = all(v for k, v in cons_available.items() if k in items_essential) if co_administrated: @@ -1230,6 +1236,18 @@ def apply(self, person_id, squeeze_factor): ) _new_contraceptive = self.new_contraceptive + + # Add used equipment + if _new_contraceptive == 'female_sterilization': + self.add_equipment({ + 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Lamp, Anglepoise' + }) + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Minor Surgery')) + elif _new_contraceptive == 'IUD': + self.add_equipment({ + 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Sponge Holding Forceps' + }) + else: _new_contraceptive = "not_using" diff --git a/src/tlo/methods/copd.py b/src/tlo/methods/copd.py index dc85f2b5d9..53602505ae 100644 --- a/src/tlo/methods/copd.py +++ b/src/tlo/methods/copd.py @@ -591,6 +591,7 @@ def apply(self, person_id, squeeze_factor): oxygen=self.get_consumables({self.module.item_codes['oxygen']: 23_040}), aminophylline=self.get_consumables({self.module.item_codes['aminophylline']: 600}) ) + self.add_equipment({'Oxygen cylinder, with regulator', 'Nasal Prongs', 'Drip stand', 'Infusion pump'}) if prob_treatment_success: df.at[person_id, 'ch_will_die_this_episode'] = False diff --git a/src/tlo/methods/diarrhoea.py b/src/tlo/methods/diarrhoea.py index 8ca36ebedb..06c8a37b18 100644 --- a/src/tlo/methods/diarrhoea.py +++ b/src/tlo/methods/diarrhoea.py @@ -13,7 +13,6 @@ Outstanding Issues * To include rotavirus vaccine - * See todo """ from __future__ import annotations @@ -1572,6 +1571,8 @@ def apply(self, person_id, squeeze_factor): if not df.at[person_id, 'is_alive']: return + self.add_equipment({'Infusion pump', 'Drip stand'}) + self.module.do_treatment(person_id=person_id, hsi_event=self) diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index dd86f91108..e00bf030fd 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -1,6 +1,6 @@ import warnings from collections import defaultdict -from typing import Counter, Iterable, Literal, Set, Union +from typing import Counter, Dict, Iterable, Literal, Set, Union import numpy as np import pandas as pd @@ -77,6 +77,7 @@ def __init__( # - Data structures for quick look-ups for items and descriptors self._item_code_lookup = self.catalogue.set_index('Item_Description')['Item_Code'].to_dict() + self._pkg_lookup = self._create_pkg_lookup() self._all_item_descriptors = set(self._item_code_lookup.keys()) self._all_item_codes = set(self._item_code_lookup.values()) self._all_fac_ids = self.master_facilities_list['Facility_ID'].unique() @@ -134,11 +135,11 @@ def parse_items(self, items: Union[int, str, Iterable[int], Iterable[str]]) -> S def check_item_codes_recognised(item_codes: set[int]): if not item_codes.issubset(self._all_item_codes): - warnings.warn(f'Item code(s) "{item_codes}" not recognised.') + warnings.warn(f'At least one item code was unrecognised: "{item_codes}".') def check_item_descriptors_recognised(item_descriptors: set[str]): if not item_descriptors.issubset(self._all_item_descriptors): - warnings.warn(f'Item descriptor(s) "{item_descriptors}" not recognised.') + warnings.warn(f'At least one item descriptor was unrecognised "{item_descriptors}".') # Make into a set if it is not one already if isinstance(items, (str, int)): @@ -248,13 +249,41 @@ def set_of_keys_or_empty_set(x: Union[set, dict]): data=row.to_dict(), ) - def lookup_item_codes_from_pkg_name(self, pkg_name: str) -> Set[int]: - """Convenience function to find the set of item_codes that are grouped under a package name in the catalogue. - It is expected that this is used by the disease module once and then the resulting equipment item_codes are - saved on the module.""" + def from_pkg_names(self, pkg_names: Union[str, Iterable[str]]) -> Set[int]: + """Convenience function to find the set of item_codes that are grouped under requested package name(s) in the + catalogue.""" + # Make into a set if it is not one already + if isinstance(pkg_names, (str, int)): + pkg_names = set([pkg_names]) + else: + pkg_names = set(pkg_names) + + item_codes = set() + for pkg_name in pkg_names: + if pkg_name in self._pkg_lookup.keys(): + item_codes.update(self._pkg_lookup[pkg_name]) + else: + raise ValueError(f'That Pkg_Name is not in the catalogue: {pkg_name=}') + + return item_codes + + def _create_pkg_lookup(self) -> Dict[str, Set[int]]: + """Create a lookup from a Package Name to a set of Item_Codes that are contained with that package. + N.B. In the Catalogue, there is one row for each Item, and the Packages to which each Item belongs (if any) + is given in a column 'Pkg_Name': if an item belongs to multiple packages, these names are separated by commas, + and if it doesn't belong to any package, then there is a NULL value.""" df = self.catalogue - if pkg_name not in df['Pkg_Name'].unique(): - raise ValueError(f'That Pkg_Name is not in the catalogue: {pkg_name=}') + # Make dataframe with columns for each package, and bools showing whether each item_code is included + pkgs = df['Pkg_Name'].replace({float('nan'): None}) \ + .str.get_dummies(sep=',') \ + .set_index(df.Item_Code) \ + .astype(bool) + + # Make dict of the form: {'Pkg_Code': } + pkg_lookup_dict = { + pkg_name.strip(): set(pkgs[pkg_name].loc[pkgs[pkg_name]].index.to_list()) + for pkg_name in pkgs.columns + } - return set(df.loc[df['Pkg_Name'] == pkg_name, 'Item_Code'].values) + return pkg_lookup_dict diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py index 591ccc6e3d..d86c706217 100644 --- a/src/tlo/methods/hiv.py +++ b/src/tlo/methods/hiv.py @@ -2419,6 +2419,10 @@ def apply(self, person_id, squeeze_factor): # Update circumcision state df.at[person_id, "li_is_circ"] = True + # Add used equipment + self.add_equipment({'Drip stand', 'Stool, adjustable height', 'Autoclave', + 'Bipolar Diathermy Machine', 'Bed, adult', 'Trolley, patient'}) + # Schedule follow-up appts # schedule first follow-up appt, 3 days from procedure; self.sim.modules["HealthSystem"].schedule_hsi_event( diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index c924f017cc..695dbeb501 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -1889,6 +1889,9 @@ def refer_for_cs(): hsi_event=hsi_event) if avail and sf_check: + # Add used equipment + hsi_event.add_equipment({'Delivery Forceps', 'Vacuum extractor'}) + pregnancy_helper_functions.log_met_need(self, f'avd_{indication}', hsi_event) # If AVD was successful then we record the mode of delivery. We use this variable to reduce @@ -2140,16 +2143,20 @@ def surgical_management_of_pph(self, hsi_event): sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='surg', hsi_event=hsi_event) - # determine if uterine preserving surgery will be successful - treatment_success_pph = params['success_rate_pph_surgery'] > self.rng.random_sample() + if avail and sf_check: + # Add used equipment + hsi_event.add_equipment(hsi_event.healthcare_system.equipment.from_pkg_names('Major Surgery')) - # If resources are available and the surgery is a success then a hysterectomy does not occur - if treatment_success_pph and avail and sf_check: - self.pph_treatment.set(person_id, 'surgery') + # determine if uterine preserving surgery will be successful + treatment_success_pph = params['success_rate_pph_surgery'] > self.rng.random_sample() - elif not treatment_success_pph and avail and sf_check: - self.pph_treatment.set(person_id, 'hysterectomy') - df.at[person_id, 'la_has_had_hysterectomy'] = True + if treatment_success_pph: + self.pph_treatment.set(person_id, 'surgery') + else: + # If the treatment is unsuccessful then women will require a hysterectomy to stop the bleeding + hsi_event.add_equipment({'Hysterectomy set'}) + self.pph_treatment.set(person_id, 'hysterectomy') + df.at[person_id, 'la_has_had_hysterectomy'] = True # log intervention delivery if self.pph_treatment.has_all(person_id, 'surgery') or df.at[person_id, 'la_has_had_hysterectomy']: @@ -2178,6 +2185,8 @@ def blood_transfusion(self, hsi_event): hsi_event=hsi_event) if avail and sf_check: + hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) + mni[person_id]['received_blood_transfusion'] = True pregnancy_helper_functions.log_met_need(self, 'blood_tran', hsi_event) @@ -2201,6 +2210,9 @@ def assessment_and_treatment_of_anaemia(self, hsi_event): mother = df.loc[person_id] mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info + # Add used equipment + hsi_event.add_equipment({'Analyser, Haematology'}) + # Use dx_test function to assess anaemia status test_result = self.sim.modules['HealthSystem'].dx_manager.run_dx_test( dx_tests_to_run='full_blood_count_hb_pn', hsi_event=hsi_event) @@ -2927,6 +2939,11 @@ def apply(self, person_id, squeeze_factor): cons=self.module.item_codes_lab_consumables['delivery_core'], opt_cons=self.module.item_codes_lab_consumables['delivery_optional']) + # Add used equipment + self.add_equipment({'Delivery set', 'Weighing scale', 'Stethoscope, foetal, monaural, Pinard, plastic', + 'Resuscitaire', 'Sphygmomanometer', 'Tray, emergency', 'Suction machine', + 'Thermometer', 'Drip stand', 'Infusion pump'}) + # If the clean delivery kit consumable is available, we assume women benefit from clean delivery if avail: mni[person_id]['clean_birth_practices'] = True @@ -3233,6 +3250,9 @@ def apply(self, person_id, squeeze_factor): logger.debug(key='message', data="cs delivery blocked for this analysis") elif (avail and sf_check) or (mni[person_id]['cs_indication'] == 'other'): + # If intervention is delivered - add used equipment + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery')) + person = df.loc[person_id] logger.info(key='caesarean_delivery', data=person.to_dict()) logger.info(key='cs_indications', data={'id': person_id, @@ -3266,6 +3286,7 @@ def apply(self, person_id, squeeze_factor): # Unsuccessful repair will lead to this woman requiring a hysterectomy. Hysterectomy will also reduce # risk of death from uterine rupture but leads to permanent infertility in the simulation else: + self.add_equipment({'Hysterectomy set'}) df.at[person_id, 'la_has_had_hysterectomy'] = True # ============================= SURGICAL MANAGEMENT OF POSTPARTUM HAEMORRHAGE================================== diff --git a/src/tlo/methods/malaria.py b/src/tlo/methods/malaria.py index 50ad58db8b..bf7b5a11be 100644 --- a/src/tlo/methods/malaria.py +++ b/src/tlo/methods/malaria.py @@ -1215,6 +1215,10 @@ def apply(self, person_id, squeeze_factor): df.at[person_id, 'ma_date_tx'] = self.sim.date df.at[person_id, 'ma_tx_counter'] += 1 + # Add used equipment + self.add_equipment({'Drip stand', 'Haemoglobinometer', + 'Analyser, Combined Chemistry and Electrolytes'}) + # rdt is offered as part of the treatment package # Log the test: line-list of summary information about each test fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id) diff --git a/src/tlo/methods/measles.py b/src/tlo/methods/measles.py index 5de96c9883..b6955ff9d7 100644 --- a/src/tlo/methods/measles.py +++ b/src/tlo/methods/measles.py @@ -460,6 +460,9 @@ def apply(self, person_id, squeeze_factor): logger.debug(key="HSI_Measles_Treatment", data=f"HSI_Measles_Treatment: giving required measles treatment to person {person_id}") + if "respiratory_symptoms" in symptoms: + self.add_equipment({'Oxygen concentrator', 'Oxygen cylinder, with regulator'}) + # modify person property which is checked when scheduled death occurs (or shouldn't occur) df.at[person_id, "me_on_treatment"] = True diff --git a/src/tlo/methods/newborn_outcomes.py b/src/tlo/methods/newborn_outcomes.py index 492c37a1fe..433b21ca88 100644 --- a/src/tlo/methods/newborn_outcomes.py +++ b/src/tlo/methods/newborn_outcomes.py @@ -987,6 +987,7 @@ def assessment_and_treatment_newborn_sepsis(self, hsi_event, facility_type): if avail and sf_check: df.at[person_id, 'nb_supp_care_neonatal_sepsis'] = True pregnancy_helper_functions.log_met_need(self, 'neo_sep_supportive_care', hsi_event) + hsi_event.add_equipment({'Drip stand', 'Infusion pump'}) # The same pattern is then followed for health centre care else: @@ -998,6 +999,7 @@ def assessment_and_treatment_newborn_sepsis(self, hsi_event, facility_type): if avail and sf_check: df.at[person_id, 'nb_inj_abx_neonatal_sepsis'] = True pregnancy_helper_functions.log_met_need(self, 'neo_sep_abx', hsi_event) + hsi_event.add_equipment({'Drip stand', 'Infusion pump', 'Oxygen cylinder, with regulator'}) def link_twins(self, child_one, child_two, mother_id): """ diff --git a/src/tlo/methods/oesophagealcancer.py b/src/tlo/methods/oesophagealcancer.py index 104770a15f..1961aa340e 100644 --- a/src/tlo/methods/oesophagealcancer.py +++ b/src/tlo/methods/oesophagealcancer.py @@ -688,13 +688,15 @@ def apply(self, person_id, squeeze_factor): return hs.get_blank_appt_footprint() # Check the consumables are available - # todo: replace with endoscope? - cons_avail = self.get_consumables(item_codes=self.module.item_codes_oesophageal_can['screening_biopsy_core'], + cons_avail = self.get_consumables(item_codes=self.module.item_codes_oesophageal_can['screening_endoscopy_core'], optional_item_codes= - self.module.item_codes_oesophageal_can['screening_biopsy_optional']) + self.module.item_codes_oesophageal_can[ + 'screening_biopsy_endoscopy_cystoscopy_optional']) if cons_avail: - # If consumables are available, run the dx_test representing the biopsy + # If consumables are available add used equipment and run the dx_test representing the biopsy + # n.b. endoscope not in equipment list + self.add_equipment({'Endoscope', 'Ordinary Microscope'}) # Use an endoscope to diagnose whether the person has Oesophageal Cancer: dx_result = hs.dx_manager.run_dx_test( @@ -783,7 +785,8 @@ def apply(self, person_id, squeeze_factor): self.module.item_codes_oesophageal_can['treatment_surgery_optional']) if cons_avail: - # If consumables are available and the treatment will go ahead + # If consumables are available and the treatment will go ahead - update the equipment + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery')) # Log chemotherapy consumables self.get_consumables( @@ -892,7 +895,8 @@ def apply(self, person_id, squeeze_factor): item_codes=self.module.item_codes_oesophageal_can['palliation']) if cons_available: - # If consumables are available and the treatment will go ahead + # If consumables are available and the treatment will go ahead - update the equipment + self.add_equipment({'Infusion pump', 'Drip stand'}) # Record the start of palliative care if this is first appointment if pd.isnull(df.at[person_id, "oc_date_palliative_care"]): diff --git a/src/tlo/methods/other_adult_cancers.py b/src/tlo/methods/other_adult_cancers.py index 774b809cfc..5999792393 100644 --- a/src/tlo/methods/other_adult_cancers.py +++ b/src/tlo/methods/other_adult_cancers.py @@ -694,10 +694,12 @@ def apply(self, person_id, squeeze_factor): # Check consumables are available cons_avail = self.get_consumables(item_codes=self.module.item_codes_other_can['screening_biopsy_core'], optional_item_codes= - self.module.item_codes_other_can['screening_biopsy_optional']) + self.module.item_codes_other_can[ + 'screening_biopsy_endoscopy_cystoscopy_optional']) if cons_avail: - # If consumables are available, run the dx_test representing the biopsy + # If consumables are available add used equipment and run the dx_test representing the biopsy + self.add_equipment({'Ultrasound scanning machine', 'Ordinary Microscope'}) # Use a diagnostic_device to diagnose whether the person has other adult cancer: dx_result = hs.dx_manager.run_dx_test( @@ -786,7 +788,8 @@ def apply(self, person_id, squeeze_factor): ) if cons_available: - # If consumables are available and the treatment will go ahead + # If consumables are available and the treatment will go ahead - update the equipment + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery')) # Record date and stage of starting treatment df.at[person_id, "oac_date_treatment"] = self.sim.date @@ -897,7 +900,8 @@ def apply(self, person_id, squeeze_factor): item_codes=self.module.item_codes_other_can['palliation']) if cons_available: - # If consumables are available and the treatment will go ahead + # If consumables are available and the treatment will go ahead - update the equipment + self.add_equipment({'Infusion pump', 'Drip stand'}) # Record the start of palliative care if this is first appointment if pd.isnull(df.at[person_id, "oac_date_palliative_care"]): diff --git a/src/tlo/methods/postnatal_supervisor.py b/src/tlo/methods/postnatal_supervisor.py index 5d99968cef..25bce6013f 100644 --- a/src/tlo/methods/postnatal_supervisor.py +++ b/src/tlo/methods/postnatal_supervisor.py @@ -1294,6 +1294,8 @@ def apply(self, person_id, squeeze_factor): self.get_consumables(item_codes=of_repair_cons) + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery')) + # Log the end of disability in the MNI pregnancy_helper_functions.store_dalys_in_mni( person_id, self.sim.modules['PregnancySupervisor'].mother_and_newborn_info, diff --git a/src/tlo/methods/prostate_cancer.py b/src/tlo/methods/prostate_cancer.py index dabb2e0593..8bb7fd82ef 100644 --- a/src/tlo/methods/prostate_cancer.py +++ b/src/tlo/methods/prostate_cancer.py @@ -782,7 +782,6 @@ def apply(self, person_id, squeeze_factor): hsi_event=self ) - # TODO: replace with PSA test when added to cons list cons_avail = self.get_consumables(item_codes=self.module.item_codes_prostate_can['screening_psa_test_optional']) if dx_result and cons_avail: @@ -824,9 +823,12 @@ def apply(self, person_id, squeeze_factor): cons_available = self.get_consumables(item_codes=self.module.item_codes_prostate_can['screening_biopsy_core'], optional_item_codes=self.module.item_codes_prostate_can[ - 'screening_biopsy_optional']) + 'screening_biopsy_endoscopy_cystoscopy_optional']) if cons_available: + # If consumables are available update the use of equipment and run the dx_test representing the biopsy + self.add_equipment({'Ultrasound scanning machine', 'Ordinary Microscope'}) + # Use a biopsy to assess whether the person has prostate cancer: dx_result = hs.dx_manager.run_dx_test( dx_tests_to_run='biopsy_for_prostate_cancer', @@ -914,7 +916,8 @@ def apply(self, person_id, squeeze_factor): 'treatment_surgery_optional']) if cons_available: - # If consumables are available the treatment will go ahead + # If consumables are available and the treatment will go ahead - update the equipment + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery')) # Record date and stage of starting treatment df.at[person_id, "pc_date_treatment"] = self.sim.date @@ -1018,7 +1021,8 @@ def apply(self, person_id, squeeze_factor): item_codes=self.module.item_codes_prostate_can['palliation']) if cons_available: - # If consumables are available and the treatment will go ahead + # If consumables are available and the treatment will go ahead - update the equipment + self.add_equipment({'Infusion pump', 'Drip stand'}) # Record the start of palliative care if this is first appointment if pd.isnull(df.at[person_id, "pc_date_palliative_care"]): diff --git a/src/tlo/methods/rti.py b/src/tlo/methods/rti.py index d6aa1d693f..b76fb40e9f 100644 --- a/src/tlo/methods/rti.py +++ b/src/tlo/methods/rti.py @@ -3213,8 +3213,13 @@ def apply(self, person_id, squeeze_factor): self.sim.population.props.at[person_id, 'rt_diagnosed'] = True road_traffic_injuries = self.sim.modules['RTI'] road_traffic_injuries.rti_injury_diagnosis(person_id, self.EXPECTED_APPT_FOOTPRINT) - if 'Tomography' in list(self.EXPECTED_APPT_FOOTPRINT.keys()): + + if 'DiagRadio' in list(self.EXPECTED_APPT_FOOTPRINT.keys()): + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('X-ray')) + + elif 'Tomography' in list(self.EXPECTED_APPT_FOOTPRINT.keys()): self.ACCEPTED_FACILITY_LEVEL = '3' + self.add_equipment({'Computed Tomography (CT machine)', 'CT scanner accessories'}) def did_not_run(self, *args, **kwargs): pass @@ -3516,6 +3521,9 @@ def apply(self, person_id, squeeze_factor): # determine the number of ICU days used to treat patient if df.loc[person_id, 'rt_ISS_score'] > self.hdu_cut_off_iss_score: + + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ICU')) + mean_icu_days = p['mean_icu_days'] sd_icu_days = p['sd_icu_days'] mean_tbi_icu_days = p['mean_tbi_icu_days'] @@ -3808,8 +3816,6 @@ class HSI_RTI_Shock_Treatment(HSI_Event, IndividualScopeEventMixin): """ This HSI event handles the process of treating hypovolemic shock, as recommended by the pediatric handbook for Malawi and (TODO: FIND ADULT REFERENCE) - Currently this HSI_Event is described only and not used, as I still need to work out how to model the occurrence - of shock """ def __init__(self, module, person_id): @@ -3857,6 +3863,7 @@ def apply(self, person_id, squeeze_factor): logger.debug(key='rti_general_message', data=f"Hypovolemic shock treatment available for person {person_id}") df.at[person_id, 'rt_in_shock'] = False + self.add_equipment({'Infusion pump', 'Drip stand', 'Oxygen cylinder, with regulator', 'Nasal Prongs'}) else: self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) return self.make_appt_footprint({}) @@ -3956,6 +3963,9 @@ def apply(self, person_id, squeeze_factor): data=f"Fracture casts available for person %d's {fracturecastcounts + slingcounts} fractures, " f"{person_id}" ) + + self.add_equipment({'Casting platform', 'Casting chairs', 'Bucket, 10L'}) + # update the property rt_med_int to indicate they are recieving treatment df.at[person_id, 'rt_med_int'] = True # Find the persons injuries @@ -4092,6 +4102,9 @@ def apply(self, person_id, squeeze_factor): logger.debug(key='rti_general_message', data=f"Fracture casts available for person {person_id} {open_fracture_counts} open fractures" ) + + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery')) + person = df.loc[person_id] # update the dataframe to show this person is recieving treatment df.loc[person_id, 'rt_med_int'] = True @@ -4257,6 +4270,7 @@ def __init__(self, module, person_id): p = self.module.parameters self.prob_mild_burns = p['prob_mild_burns'] + def apply(self, person_id, squeeze_factor): get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name df = self.sim.population.props @@ -4810,6 +4824,9 @@ def apply(self, person_id, squeeze_factor): # RTI_Med assert df.loc[person_id, 'rt_diagnosed'], 'This person has not been through a and e' assert df.loc[person_id, 'rt_med_int'], 'This person has not been through rti med int' + + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery')) + # ------------------------ Track permanent disabilities with treatment ------------------------------------- # --------------------------------- Perm disability from TBI ----------------------------------------------- codes = ['133', '133a', '133b', '133c', '133d', '134', '134a', '134b', '135'] @@ -5125,6 +5142,8 @@ def apply(self, person_id, squeeze_factor): # todo: think about consequences of certain consumables not being available for minor surgery and model health # outcomes if request_outcome: + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery')) + # create a dictionary to store the recovery times for each injury in days minor_surg_recov_time_days = { '322': 180, diff --git a/src/tlo/methods/schisto.py b/src/tlo/methods/schisto.py index 810c869f10..0e9735286a 100644 --- a/src/tlo/methods/schisto.py +++ b/src/tlo/methods/schisto.py @@ -923,6 +923,8 @@ def apply(self, person_id, squeeze_factor): ) if will_test: + self.add_equipment({'Ordinary Microscope'}) + # Determine if they truly are infected (with any of the species) is_infected = (person.loc[cols_of_infection_status] != 'Non-infected').any() diff --git a/src/tlo/methods/symptommanager.py b/src/tlo/methods/symptommanager.py index 68edbf0840..26f6aa7ee4 100644 --- a/src/tlo/methods/symptommanager.py +++ b/src/tlo/methods/symptommanager.py @@ -272,7 +272,8 @@ def pre_initialise_population(self): SymptomManager.PROPERTIES = dict() for symptom_name in sorted(self.symptom_names): symptom_column_name = self.get_column_name_for_symptom(symptom_name) - SymptomManager.PROPERTIES[symptom_column_name] = Property(Types.BITSET, f'Presence of symptom {symptom_name}') + SymptomManager.PROPERTIES[symptom_column_name] = Property(Types.BITSET, + f'Presence of symptom {symptom_name}') def initialise_population(self, population): """ diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py index 053469a253..ddd2f9af43 100644 --- a/src/tlo/methods/tb.py +++ b/src/tlo/methods/tb.py @@ -1706,6 +1706,9 @@ def apply(self, person_id, squeeze_factor): ].dx_manager.run_dx_test( dx_tests_to_run="tb_clinical", hsi_event=self ) + if test_result is not None: + # Add used equipment + self.add_equipment({'Sputum Collection box', 'Ordinary Microscope'}) elif test == "xpert": @@ -1738,6 +1741,9 @@ def apply(self, person_id, squeeze_factor): dx_tests_to_run="tb_xpert_test_smear_negative", hsi_event=self, ) + if test_result is not None: + # Add used equipment + self.add_equipment({'Sputum Collection box', 'Gene Expert (16 Module)'}) # ------------------------- testing referrals ------------------------- # @@ -1756,6 +1762,9 @@ def apply(self, person_id, squeeze_factor): ACTUAL_APPT_FOOTPRINT = self.make_appt_footprint( {"Over5OPD": 2, "LabTBMicro": 1} ) + if test_result is not None: + # Add used equipment + self.add_equipment({'Sputum Collection box', 'Ordinary Microscope'}) # if still no result available, rely on clinical diagnosis if test_result is None: @@ -1953,6 +1962,8 @@ def apply(self, person_id, squeeze_factor): test_result = self.sim.modules["HealthSystem"].dx_manager.run_dx_test( dx_tests_to_run="tb_xray_smear_negative", hsi_event=self ) + if test_result is not None: + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('X-ray')) # if consumables not available, refer to level 2 # return blank footprint as xray did not occur @@ -2026,6 +2037,8 @@ def apply(self, person_id, squeeze_factor): test_result = self.sim.modules["HealthSystem"].dx_manager.run_dx_test( dx_tests_to_run="tb_xray_smear_negative", hsi_event=self ) + if test_result is not None: + self.add_equipment(self.healthcare_system.equipment.from_pkg_names('X-ray')) # if consumables not available, rely on clinical diagnosis # return blank footprint as xray was not available @@ -2287,6 +2300,9 @@ def apply(self, person_id, squeeze_factor): test_result = self.sim.modules["HealthSystem"].dx_manager.run_dx_test( dx_tests_to_run="tb_sputum_test_smear_negative", hsi_event=self ) + if test_result is not None: + # Add used equipment + self.add_equipment({'Sputum Collection box', 'Ordinary Microscope'}) # if sputum test was available and returned positive and not diagnosed with mdr, schedule xpert test if test_result and not person["tb_diagnosed_mdr"]: @@ -2301,6 +2317,9 @@ def apply(self, person_id, squeeze_factor): xperttest_result = self.sim.modules["HealthSystem"].dx_manager.run_dx_test( dx_tests_to_run="tb_xpert_test_smear_negative", hsi_event=self ) + if xperttest_result is not None: + # Add used equipment + self.add_equipment({'Sputum Collection box', 'Gene Expert (16 Module)'}) # if xpert test returns new mdr-tb diagnosis if xperttest_result and (df.at[person_id, "tb_strain"] == "mdr"): diff --git a/tests/test_equipment.py b/tests/test_equipment.py index a02ea282f8..1167023aa8 100644 --- a/tests/test_equipment.py +++ b/tests/test_equipment.py @@ -22,14 +22,18 @@ def test_core_functionality_of_equipment_class(seed): # Create toy data catalogue = pd.DataFrame( + # PkgWith0+1 stands alone or as multiple pkgs for one item; PkgWith1 is only as multiple pkgs + # for one item; PkgWith3 only stands alone [ {"Item_Description": "ItemZero", "Item_Code": 0, "Pkg_Name": 'PkgWith0+1'}, - {"Item_Description": "ItemOne", "Item_Code": 1, "Pkg_Name": 'PkgWith0+1'}, + {"Item_Description": "ItemOne", "Item_Code": 1, "Pkg_Name": 'PkgWith0+1, PkgWith1'}, {"Item_Description": "ItemTwo", "Item_Code": 2, "Pkg_Name": float('nan')}, + {"Item_Description": "ItemThree", "Item_Code": 3, "Pkg_Name": 'PkgWith3'}, ] ) data_availability = pd.DataFrame( - # item 0 is not available anywhere; item 1 is available everywhere; item 2 is available only at facility_id=1 + # item 0 is not available anywhere; item 1 is available everywhere; item 2 is available only at facility_id=1; + # item 3 is available only at facility_id=0 [ {"Item_Code": 0, "Facility_ID": 0, "Pr_Available": 0.0}, {"Item_Code": 0, "Facility_ID": 1, "Pr_Available": 0.0}, @@ -37,6 +41,8 @@ def test_core_functionality_of_equipment_class(seed): {"Item_Code": 1, "Facility_ID": 1, "Pr_Available": 1.0}, {"Item_Code": 2, "Facility_ID": 0, "Pr_Available": 0.0}, {"Item_Code": 2, "Facility_ID": 1, "Pr_Available": 1.0}, + {"Item_Code": 3, "Facility_ID": 0, "Pr_Available": 1.0}, + {"Item_Code": 3, "Facility_ID": 1, "Pr_Available": 0.0}, ] ) mfl = pd.DataFrame( @@ -76,6 +82,22 @@ def test_core_functionality_of_equipment_class(seed): with pytest.warns(): eq_default.parse_items('ItemThatIsNotDefined') + # Lookup the item_codes that belong in a particular package. + # - When package is recognised + # if items are in the same package (once standing alone, once within multiple pkgs defined for item) + assert {0, 1} == eq_default.from_pkg_names(pkg_names='PkgWith0+1') + # if the pkg within multiple pkgs defined for item + assert {1} == eq_default.from_pkg_names(pkg_names='PkgWith1') + # if the pkg only stands alone + assert {3} == eq_default.from_pkg_names(pkg_names='PkgWith3') + # Lookup the item_codes that belong to multiple specified packages. + assert {0, 1, 3} == eq_default.from_pkg_names(pkg_names={'PkgWith0+1', 'PkgWith3'}) + assert {1, 3} == eq_default.from_pkg_names(pkg_names={'PkgWith1', 'PkgWith3'}) + + # - When package is not recognised (should raise an error) + with pytest.raises(ValueError): + eq_default.from_pkg_names(pkg_names='') + # Testing checking on available of items # - calling when all items available (should be true) assert eq_default.is_all_items_available(item_codes={1, 2}, facility_id=1) @@ -132,19 +154,11 @@ def test_core_functionality_of_equipment_class(seed): # - Check that internal record is as expected assert {0: {0: 1, 1: 2}, 1: {0: 1, 1: 1}} == dict(eq_default._record_of_equipment_used_by_facility_id) - # Lookup the item_codes that belong in a particular package. - # - When package is recognised - assert {0, 1} == eq_default.lookup_item_codes_from_pkg_name(pkg_name='PkgWith0+1') # these items are in the same - # package - # - Error thrown when package is not recognised - with pytest.raises(ValueError): - eq_default.lookup_item_codes_from_pkg_name(pkg_name='') - - equipment_item_code_that_is_available = [0, 1, ] equipment_item_code_that_is_not_available = [2, 3,] + def run_simulation_and_return_log( seed, tmpdir, equipment_in_init, equipment_in_apply ) -> Dict: From 3348552fd92de20618948d23d26fb99adc2246da Mon Sep 17 00:00:00 2001 From: Tara <37845078+tdm32@users.noreply.github.com> Date: Sun, 23 Jun 2024 13:03:20 +0100 Subject: [PATCH 04/19] Reduce the Excessive Demand for TB Appointment Under the 'Max HealthSystem Functionality' Scenario Switch (#1400) * edit ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking TB: rate_testing_general_pop removed as this refers to general population screening NTP2019 update treatment_coverage to 99.99 for all years Schisto: delay until HSIs repeated was 0.999, should be integer value of 1 day (default is 5 days) HSI_Tb_ScreeningAndRefer added condition if tested within last 2 weeks, do nothing * replace 1 (delays in days) with 1.00001 The parameter type is defined as REAL, so this is the only way I've found to make Excel store it as a float (and not an int of 1 that would get coerned into a bool!). Where this is used (line 944 in schisto.py) there is a coecision to int at the last minute so this should presevre the intended behaviour * Update src/tlo/methods/tb.py --------- Co-authored-by: Tim Hallett <39991060+tbhallett@users.noreply.github.com> --- resources/ResourceFile_HIV.xlsx | 4 ++-- ...d_Healthsystem_And_Healthcare_Seeking.xlsx | 4 ++-- resources/ResourceFile_TB.xlsx | 2 +- src/tlo/methods/tb.py | 21 ++++++++++++------- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/resources/ResourceFile_HIV.xlsx b/resources/ResourceFile_HIV.xlsx index 64ef25c261..1cdb865eb1 100644 --- a/resources/ResourceFile_HIV.xlsx +++ b/resources/ResourceFile_HIV.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e2345032931c1360046dc7394681cc39669687888f7f8f3e42469d8add067438 -size 160376 +oid sha256:58978c108515c3762addd18824129b2654f241d94bcc778ab17b27d0d8250593 +size 160402 diff --git a/resources/ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx b/resources/ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx index ff88584e3f..8fc0a24ae9 100644 --- a/resources/ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx +++ b/resources/ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:da900375dda86f999e744bfb6d6dec7347d5b13f176046ba182740421a43d256 -size 48274 +oid sha256:1b462c20ca6cbf0ca1f98936416e015fa248289e5bf4f66838e1b9920874f651 +size 48142 diff --git a/resources/ResourceFile_TB.xlsx b/resources/ResourceFile_TB.xlsx index d40eb40490..e6c1bf80db 100644 --- a/resources/ResourceFile_TB.xlsx +++ b/resources/ResourceFile_TB.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3fc295cc70b8e86c75e1725a92ceb95bc86a26d3fbe1f680db379726bcab3ab3 +oid sha256:3cb13e128d4bcb3b694def108c3bd61b16508b48e389c3e5cdf8155717aab9e9 size 55662 diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py index ddd2f9af43..02d860fe52 100644 --- a/src/tlo/methods/tb.py +++ b/src/tlo/methods/tb.py @@ -1467,7 +1467,7 @@ def apply(self, population): ) # -------- 5) schedule screening for asymptomatic and symptomatic people -------- - # sample from all new active cases (active_idx) and determine whether they will seek a test + # sample from all NEW active cases (active_idx) and determine whether they will seek a test year = min(2019, max(2011, now.year)) active_testing_rates = p["rate_testing_active_tb"] @@ -1610,8 +1610,19 @@ def apply(self, person_id, squeeze_factor): p = self.module.parameters person = df.loc[person_id] - # If the person is dead or already diagnosed, do nothing do not occupy any resources - if not person["is_alive"] or person["tb_diagnosed"]: + if not person["is_alive"]: + return self.sim.modules["HealthSystem"].get_blank_appt_footprint() + + # If the person is already diagnosed, do nothing do not occupy any resources + if person["tb_diagnosed"]: + return self.sim.modules["HealthSystem"].get_blank_appt_footprint() + + # If the person is already on treatment and not failing, do nothing do not occupy any resources + if person["tb_on_treatment"] and not person["tb_treatment_failure"]: + return self.sim.modules["HealthSystem"].get_blank_appt_footprint() + + # if person has tested within last 14 days, do nothing + if person["tb_date_tested"] >= (self.sim.date - DateOffset(days=7)): return self.sim.modules["HealthSystem"].get_blank_appt_footprint() logger.debug( @@ -1620,10 +1631,6 @@ def apply(self, person_id, squeeze_factor): smear_status = person["tb_smear"] - # If the person is already on treatment and not failing, do nothing do not occupy any resources - if person["tb_on_treatment"] and not person["tb_treatment_failure"]: - return self.sim.modules["HealthSystem"].get_blank_appt_footprint() - # ------------------------- screening ------------------------- # # check if patient has: cough, fever, night sweat, weight loss From a83f624e56817fe0c37d7b0e1db55cf593ccb63e Mon Sep 17 00:00:00 2001 From: Tara <37845078+tdm32@users.noreply.github.com> Date: Thu, 27 Jun 2024 19:57:59 +0100 Subject: [PATCH 05/19] Scenario switcher - htm (#1377) --- resources/ResourceFile_HIV.xlsx | 4 +- resources/ResourceFile_TB.xlsx | 4 +- resources/malaria/ResourceFile_malaria.xlsx | 4 +- .../analysis_htm_scaleup.py | 112 ++++++++++ .../htm_scenario_analyses/scenario_plots.py | 139 ++++++++++++ src/scripts/malaria/analysis_malaria.py | 15 +- src/tlo/methods/hiv.py | 76 ++++++- src/tlo/methods/malaria.py | 102 ++++++++- src/tlo/methods/tb.py | 65 +++++- tests/test_htm_scaleup.py | 210 ++++++++++++++++++ 10 files changed, 714 insertions(+), 17 deletions(-) create mode 100644 src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py create mode 100644 src/scripts/htm_scenario_analyses/scenario_plots.py create mode 100644 tests/test_htm_scaleup.py diff --git a/resources/ResourceFile_HIV.xlsx b/resources/ResourceFile_HIV.xlsx index 1cdb865eb1..f76169e701 100644 --- a/resources/ResourceFile_HIV.xlsx +++ b/resources/ResourceFile_HIV.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:58978c108515c3762addd18824129b2654f241d94bcc778ab17b27d0d8250593 -size 160402 +oid sha256:913d736db7717519270d61824a8855cbfd4d6e61a73b7ce51e2c3b7915b011ff +size 161597 diff --git a/resources/ResourceFile_TB.xlsx b/resources/ResourceFile_TB.xlsx index e6c1bf80db..3dfc69cd81 100644 --- a/resources/ResourceFile_TB.xlsx +++ b/resources/ResourceFile_TB.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3cb13e128d4bcb3b694def108c3bd61b16508b48e389c3e5cdf8155717aab9e9 -size 55662 +oid sha256:120d687122772909c267db41c933664ccc6247c8aef59d49532547c0c3791121 +size 55634 diff --git a/resources/malaria/ResourceFile_malaria.xlsx b/resources/malaria/ResourceFile_malaria.xlsx index 70902b7480..a6487e80ae 100644 --- a/resources/malaria/ResourceFile_malaria.xlsx +++ b/resources/malaria/ResourceFile_malaria.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6ba5849e265103ee799d1982325b6fed1ef4d3df559ffce9d6790395c201fcaf -size 67562 +oid sha256:e8157368754dae9ce692fbd10fecf1e598f37fb258292085c93e1c881dd47aa9 +size 69590 diff --git a/src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py b/src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py new file mode 100644 index 0000000000..a89231f670 --- /dev/null +++ b/src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py @@ -0,0 +1,112 @@ + +""" +This scenario file sets up the scenarios for simulating the effects of scaling up programs + +The scenarios are: +*0 baseline mode 1 +*1 scale-up HIV program +*2 scale-up TB program +*3 scale-up malaria program +*4 scale-up HIV and Tb and malaria programs + +scale-up occurs on the default scale-up start date (01/01/2025: in parameters list of resourcefiles) + +For all scenarios, keep all default health system settings + +check the batch configuration gets generated without error: +tlo scenario-run --draw-only src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py + +Run on the batch system using: +tlo batch-submit src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py + +or locally using: +tlo scenario-run src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py + +or execute a single run: +tlo scenario-run src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py --draw 1 0 + +""" + +from pathlib import Path + +from tlo import Date, logging +from tlo.methods import ( + demography, + enhanced_lifestyle, + epi, + healthburden, + healthseekingbehaviour, + healthsystem, + hiv, + malaria, + simplified_births, + symptommanager, + tb, +) +from tlo.scenario import BaseScenario + + +class EffectOfProgrammes(BaseScenario): + def __init__(self): + super().__init__() + self.seed = 0 + self.start_date = Date(2010, 1, 1) + self.end_date = Date(2020, 1, 1) + self.pop_size = 75_000 + self.number_of_draws = 5 + self.runs_per_draw = 1 + + def log_configuration(self): + return { + 'filename': 'scaleup_tests', + 'directory': Path('./outputs'), # <- (specified only for local running) + 'custom_levels': { + '*': logging.WARNING, + 'tlo.methods.hiv': logging.INFO, + 'tlo.methods.tb': logging.INFO, + 'tlo.methods.malaria': logging.INFO, + 'tlo.methods.demography': logging.INFO, + } + } + + def modules(self): + + return [ + demography.Demography(resourcefilepath=self.resources), + simplified_births.SimplifiedBirths(resourcefilepath=self.resources), + enhanced_lifestyle.Lifestyle(resourcefilepath=self.resources), + healthsystem.HealthSystem(resourcefilepath=self.resources), + symptommanager.SymptomManager(resourcefilepath=self.resources), + healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=self.resources), + healthburden.HealthBurden(resourcefilepath=self.resources), + epi.Epi(resourcefilepath=self.resources), + hiv.Hiv(resourcefilepath=self.resources), + tb.Tb(resourcefilepath=self.resources), + malaria.Malaria(resourcefilepath=self.resources), + ] + + def draw_parameters(self, draw_number, rng): + scaleup_start_year = 2012 + + return { + 'Hiv': { + 'do_scaleup': [False, True, False, False, True][draw_number], + 'scaleup_start_year': scaleup_start_year + }, + 'Tb': { + 'do_scaleup': [False, False, True, False, True][draw_number], + 'scaleup_start_year': scaleup_start_year + }, + 'Malaria': { + 'do_scaleup': [False, False, False, True, True][draw_number], + 'scaleup_start_year': scaleup_start_year + }, + } + + +if __name__ == '__main__': + from tlo.cli import scenario_run + + scenario_run([__file__]) + + diff --git a/src/scripts/htm_scenario_analyses/scenario_plots.py b/src/scripts/htm_scenario_analyses/scenario_plots.py new file mode 100644 index 0000000000..d14454ae13 --- /dev/null +++ b/src/scripts/htm_scenario_analyses/scenario_plots.py @@ -0,0 +1,139 @@ +""" this reads in the outputs generates through analysis_htm_scaleup.py +and produces plots for HIV, TB and malaria incidence +""" + + +import datetime +from pathlib import Path + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns + +from tlo import Date +from tlo.analysis.utils import ( + extract_params, + extract_results, + get_scenario_info, + get_scenario_outputs, + load_pickled_dataframes, +) + +resourcefilepath = Path("./resources") +datestamp = datetime.date.today().strftime("__%Y_%m_%d") + +outputspath = Path("./outputs") + + +# 0) Find results_folder associated with a given batch_file (and get most recent [-1]) +results_folder = get_scenario_outputs("scaleup_tests", outputspath)[-1] + +# Declare path for output graphs from this script +make_graph_file_name = lambda stub: results_folder / f"{stub}.png" # noqa: E731 + +# look at one log (so can decide what to extract) +log = load_pickled_dataframes(results_folder) + +# get basic information about the results +info = get_scenario_info(results_folder) + +# 1) Extract the parameters that have varied over the set of simulations +params = extract_params(results_folder) + + +# DEATHS + + +def get_num_deaths_by_cause_label(_df): + """Return total number of Deaths by label within the TARGET_PERIOD + values are summed for all ages + df returned: rows=COD, columns=draw + """ + return _df \ + .loc[pd.to_datetime(_df.date).between(*TARGET_PERIOD)] \ + .groupby(_df['label']) \ + .size() + + +TARGET_PERIOD = (Date(2015, 1, 1), Date(2020, 1, 1)) + +num_deaths_by_cause_label = extract_results( + results_folder, + module='tlo.methods.demography', + key='death', + custom_generate_series=get_num_deaths_by_cause_label, + do_scaling=True + ) + + +def summarise_deaths_for_one_cause(results_folder, label): + """ returns mean deaths for each year of the simulation + values are aggregated across the runs of each draw + for the specified cause + """ + + results_deaths = extract_results( + results_folder, + module="tlo.methods.demography", + key="death", + custom_generate_series=( + lambda df: df.assign(year=df["date"].dt.year).groupby( + ["year", "label"])["person_id"].count() + ), + do_scaling=True, + ) + # removes multi-index + results_deaths = results_deaths.reset_index() + + # select only cause specified + tmp = results_deaths.loc[ + (results_deaths.label == label) + ] + + # group deaths by year + tmp = pd.DataFrame(tmp.groupby(["year"]).sum()) + + # get mean for each draw + mean_deaths = pd.concat({'mean': tmp.iloc[:, 1:].groupby(level=0, axis=1).mean()}, axis=1).swaplevel(axis=1) + + return mean_deaths + + +aids_deaths = summarise_deaths_for_one_cause(results_folder, 'AIDS') +tb_deaths = summarise_deaths_for_one_cause(results_folder, 'TB (non-AIDS)') +malaria_deaths = summarise_deaths_for_one_cause(results_folder, 'Malaria') + +draw_labels = ['No scale-up', 'HIV, scale-up', 'TB scale-up', 'Malaria scale-up'] + +colors = sns.color_palette("Set1", 4) # Blue, Orange, Green, Red + + +# Create subplots +fig, axs = plt.subplots(3, 1, figsize=(6, 10)) + +# Plot for df1 +for i, col in enumerate(aids_deaths.columns): + axs[0].plot(aids_deaths.index, aids_deaths[col], label=draw_labels[i], color=colors[i]) +axs[0].set_title('HIV/AIDS') +axs[0].legend() +axs[0].axvline(x=2015, color='gray', linestyle='--') + +# Plot for df2 +for i, col in enumerate(tb_deaths.columns): + axs[1].plot(tb_deaths.index, tb_deaths[col], color=colors[i]) +axs[1].set_title('TB') +axs[1].axvline(x=2015, color='gray', linestyle='--') + +# Plot for df3 +for i, col in enumerate(malaria_deaths.columns): + axs[2].plot(malaria_deaths.index, malaria_deaths[col], color=colors[i]) +axs[2].set_title('Malaria') +axs[2].axvline(x=2015, color='gray', linestyle='--') + +for ax in axs: + ax.set_xlabel('Years') + ax.set_ylabel('Number deaths') + +plt.tight_layout() +plt.show() + diff --git a/src/scripts/malaria/analysis_malaria.py b/src/scripts/malaria/analysis_malaria.py index 56d05cf3ae..b2b4217dc6 100644 --- a/src/scripts/malaria/analysis_malaria.py +++ b/src/scripts/malaria/analysis_malaria.py @@ -34,8 +34,8 @@ resourcefilepath = Path("./resources") start_date = Date(2010, 1, 1) -end_date = Date(2016, 1, 1) -popsize = 300 +end_date = Date(2014, 1, 1) +popsize = 100 # set up the log config @@ -84,6 +84,15 @@ ) ) +# update parameters +sim.modules["Hiv"].parameters["do_scaleup"] = True +sim.modules["Tb"].parameters["do_scaleup"] = True +sim.modules["Malaria"].parameters["do_scaleup"] = True +sim.modules["Hiv"].parameters["scaleup_start"] = 2 +sim.modules["Tb"].parameters["scaleup_start"] = 2 +sim.modules["Malaria"].parameters["scaleup_start"] = 2 + + # Run the simulation and flush the logger sim.make_initial_population(n=popsize) sim.simulate(end_date=end_date) @@ -97,5 +106,5 @@ pickle.dump(dict(output), f, pickle.HIGHEST_PROTOCOL) # load the results -with open(outputpath / "default_run.pickle", "rb") as f: +with open(outputpath / "malaria_run.pickle", "rb") as f: output = pickle.load(f) diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py index d86c706217..1ddafe4c47 100644 --- a/src/tlo/methods/hiv.py +++ b/src/tlo/methods/hiv.py @@ -31,7 +31,7 @@ import numpy as np import pandas as pd -from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging +from tlo import DAYS_IN_YEAR, Date, DateOffset, Module, Parameter, Property, Types, logging from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata, demography, tb @@ -397,6 +397,19 @@ def __init__(self, name=None, resourcefilepath=None, run_with_checks=False): "length in days of inpatient stay for end-of-life HIV patients: list has two elements [low-bound-inclusive," " high-bound-exclusive]", ), + # ------------------ scale-up parameters for scenario analysis ------------------ # + "do_scaleup": Parameter( + Types.BOOL, + "argument to determine whether scale-up of program will be implemented" + ), + "scaleup_start_year": Parameter( + Types.INT, + "the year when the scale-up starts (it will occur on 1st January of that year)" + ), + "scaleup_parameters": Parameter( + Types.DICT, + "the parameters and values changed in scenario analysis" + ), } def read_parameters(self, data_folder): @@ -434,6 +447,9 @@ def read_parameters(self, data_folder): # Load spectrum estimates of treatment cascade p["treatment_cascade"] = workbook["spectrum_treatment_cascade"] + # load parameters for scale-up projections + p["scaleup_parameters"] = workbook["scaleup_parameters"].set_index('parameter')['scaleup_value'].to_dict() + # DALY weights # get the DALY weight that this module will use from the weight database (these codes are just random!) if "HealthBurden" in self.sim.modules.keys(): @@ -894,6 +910,12 @@ def initialise_simulation(self, sim): # 2) Schedule the Logging Event sim.schedule_event(HivLoggingEvent(self), sim.date + DateOffset(years=1)) + # Optional: Schedule the scale-up of programs + if self.parameters["do_scaleup"]: + scaleup_start_date = Date(self.parameters["scaleup_start_year"], 1, 1) + assert scaleup_start_date >= self.sim.start_date, f"Date {scaleup_start_date} is before simulation starts." + sim.schedule_event(HivScaleUpEvent(self), scaleup_start_date) + # 3) Determine who has AIDS and impose the Symptoms 'aids_symptoms' # Those on ART currently (will not get any further events scheduled): @@ -1076,6 +1098,44 @@ def initialise_simulation(self, sim): ) ) + def update_parameters_for_program_scaleup(self): + + p = self.parameters + scaled_params = p["scaleup_parameters"] + + if p["do_scaleup"]: + + # scale-up HIV program + # reduce risk of HIV - applies to whole adult population + p["beta"] = p["beta"] * scaled_params["reduction_in_hiv_beta"] + + # increase PrEP coverage for FSW after HIV test + p["prob_prep_for_fsw_after_hiv_test"] = scaled_params["prob_prep_for_fsw_after_hiv_test"] + + # prep poll for AGYW - target to the highest risk + # increase retention to 75% for FSW and AGYW + p["prob_prep_for_agyw"] = scaled_params["prob_prep_for_agyw"] + p["probability_of_being_retained_on_prep_every_3_months"] = scaled_params["probability_of_being_retained_on_prep_every_3_months"] + + # increase probability of VMMC after hiv test + p["prob_circ_after_hiv_test"] = scaled_params["prob_circ_after_hiv_test"] + + # increase testing/diagnosis rates, default 2020 0.03/0.25 -> 93% dx + p["hiv_testing_rates"]["annual_testing_rate_children"] = scaled_params["annual_testing_rate_children"] + p["hiv_testing_rates"]["annual_testing_rate_adults"] = scaled_params["annual_testing_rate_adults"] + + # ANC testing - value for mothers and infants testing + p["prob_hiv_test_at_anc_or_delivery"] = scaled_params["prob_hiv_test_at_anc_or_delivery"] + p["prob_hiv_test_for_newborn_infant"] = scaled_params["prob_hiv_test_for_newborn_infant"] + + # prob ART start if dx, this is already 95% at 2020 + p["prob_start_art_after_hiv_test"] = scaled_params["prob_start_art_after_hiv_test"] + + # viral suppression rates + # adults already at 95% by 2020 + # change all column values + p["prob_start_art_or_vs"]["virally_suppressed_on_art"] = scaled_params["virally_suppressed_on_art"] + def on_birth(self, mother_id, child_id): """ * Initialise our properties for a newborn individual; @@ -2214,6 +2274,20 @@ def apply(self, person_id): ) +class HivScaleUpEvent(Event, PopulationScopeEventMixin): + """ This event exists to change parameters or functions + depending on the scenario for projections which has been set + It only occurs once on date: scaleup_start_date, + called by initialise_simulation + """ + + def __init__(self, module): + super().__init__(module) + + def apply(self, population): + self.module.update_parameters_for_program_scaleup() + + # --------------------------------------------------------------------------- # Health System Interactions (HSI) # --------------------------------------------------------------------------- diff --git a/src/tlo/methods/malaria.py b/src/tlo/methods/malaria.py index bf7b5a11be..f322783717 100644 --- a/src/tlo/methods/malaria.py +++ b/src/tlo/methods/malaria.py @@ -11,7 +11,7 @@ import pandas as pd -from tlo import DateOffset, Module, Parameter, Property, Types, logging +from tlo import Date, DateOffset, Module, Parameter, Property, Types, logging from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, Predictor from tlo.methods import Metadata @@ -188,8 +188,20 @@ def __init__(self, name=None, resourcefilepath=None): 'prob_of_treatment_success': Parameter( Types.REAL, 'probability that treatment will clear malaria symptoms' + ), + # ------------------ scale-up parameters for scenario analysis ------------------ # + "do_scaleup": Parameter( + Types.BOOL, + "argument to determine whether scale-up of program will be implemented" + ), + "scaleup_start_year": Parameter( + Types.INT, + "the year when the scale-up starts (it will occur on 1st January of that year)" + ), + "scaleup_parameters": Parameter( + Types.DICT, + "the parameters and values changed in scenario analysis" ) - } PROPERTIES = { @@ -242,11 +254,15 @@ def read_parameters(self, data_folder): p['sev_symp_prob'] = workbook['severe_symptoms'] p['rdt_testing_rates'] = workbook['WHO_TestData2023'] + p['highrisk_districts'] = workbook['highrisk_districts'] p['inf_inc'] = pd.read_csv(self.resourcefilepath / 'malaria' / 'ResourceFile_malaria_InfInc_expanded.csv') p['clin_inc'] = pd.read_csv(self.resourcefilepath / 'malaria' / 'ResourceFile_malaria_ClinInc_expanded.csv') p['sev_inc'] = pd.read_csv(self.resourcefilepath / 'malaria' / 'ResourceFile_malaria_SevInc_expanded.csv') + # load parameters for scale-up projections + p["scaleup_parameters"] = workbook["scaleup_parameters"].set_index('parameter')['scaleup_value'].to_dict() + # check itn projected values are <=0.7 and rounded to 1dp for matching to incidence tables p['itn'] = round(p['itn'], 1) assert (p['itn'] <= 0.7) @@ -356,7 +372,7 @@ def pre_initialise_population(self): p['rr_severe_malaria_hiv_over5']), Predictor().when('(hv_inf == True) & (is_pregnant == True)', p['rr_severe_malaria_hiv_pregnant']), - ] if "hiv" in self.sim.modules else [] + ] if "Hiv" in self.sim.modules else [] self.lm["rr_of_severe_malaria"] = LinearModel.multiplicative( *(predictors + conditional_predictors)) @@ -534,8 +550,12 @@ def general_population_rdt_scheduler(self, population): # extract annual testing rates from NMCP reports # this is the # rdts issued divided by population size - test_rates = p['rdt_testing_rates'].set_index('Year')['Rate_rdt_testing'].dropna() - rdt_rate = test_rates.loc[min(test_rates.index.max(), self.sim.date.year)] / 12 + year = self.sim.date.year if self.sim.date.year <= 2024 else 2024 + + test_rates = ( + p['rdt_testing_rates'].set_index('Year')['Rate_rdt_testing'].dropna() + ) + rdt_rate = test_rates.loc[min(test_rates.index.max(), year)] / 12 # adjust rdt usage reported rate to reflect consumables availability rdt_rate = rdt_rate * p['scaling_factor_for_rdt_availability'] @@ -578,6 +598,12 @@ def initialise_simulation(self, sim): sim.schedule_event(MalariaTxLoggingEvent(self), sim.date + DateOffset(years=1)) sim.schedule_event(MalariaPrevDistrictLoggingEvent(self), sim.date + DateOffset(months=1)) + # Optional: Schedule the scale-up of programs + if self.parameters["do_scaleup"]: + scaleup_start_date = Date(self.parameters["scaleup_start_year"], 1, 1) + assert scaleup_start_date >= self.sim.start_date, f"Date {scaleup_start_date} is before simulation starts." + sim.schedule_event(MalariaScaleUpEvent(self), scaleup_start_date) + # 2) ----------------------------------- DIAGNOSTIC TESTS ----------------------------------- # Create the diagnostic test representing the use of RDT for malaria diagnosis # and registers it with the Diagnostic Test Manager @@ -626,7 +652,56 @@ def initialise_simulation(self, sim): # malaria IPTp for pregnant women self.item_codes_for_consumables_required['malaria_iptp'] = get_item_code( - 'Sulfamethoxazole + trimethropin, tablet 400 mg + 80 mg') + 'Sulfamethoxazole + trimethropin, tablet 400 mg + 80 mg' + ) + + def update_parameters_for_program_scaleup(self): + + p = self.parameters + scaled_params = p["scaleup_parameters"] + + if p["do_scaleup"]: + + # scale-up malaria program + # increase testing + # prob_malaria_case_tests=0.4 default + p["prob_malaria_case_tests"] = scaled_params["prob_malaria_case_tests"] + + # gen pop testing rates + # annual Rate_rdt_testing=0.64 at 2023 + p["rdt_testing_rates"]["Rate_rdt_testing"] = scaled_params["rdt_testing_rates"] + + # treatment reaches XX + # no default between testing and treatment, governed by tx availability + + # coverage IPTp reaches XX + # given during ANC visits and MalariaIPTp Event which selects ALL eligible women + + # treatment success reaches 1 - default is currently 1 also + p["prob_of_treatment_success"] = scaled_params["prob_of_treatment_success"] + + # bednet and ITN coverage + # set IRS for 4 high-risk districts + # lookup table created in malaria read_parameters + # produces self.itn_irs called by malaria poll to draw incidence + # need to overwrite this + highrisk_distr_num = p["highrisk_districts"]["district_num"] + + # Find indices where District_Num is in highrisk_distr_num + mask = self.itn_irs['irs_rate'].index.get_level_values('District_Num').isin( + highrisk_distr_num) + + # IRS values can be 0 or 0.8 - no other value in lookup table + self.itn_irs['irs_rate'].loc[mask] = scaled_params["irs_district"] + + # set ITN for all districts + # Set these values to 0.7 - this is the max value possible in lookup table + # equivalent to 0.7 of all pop sleeping under bednet + # household coverage could be 100%, but not everyone in household sleeping under bednet + self.itn_irs['itn_rate'] = scaled_params["itn_district"] + + # itn rates for 2019 onwards + p["itn"] = scaled_params["itn"] def on_birth(self, mother_id, child_id): df = self.sim.population.props @@ -819,6 +894,21 @@ def apply(self, population): self.module.general_population_rdt_scheduler(population) +class MalariaScaleUpEvent(Event, PopulationScopeEventMixin): + """ This event exists to change parameters or functions + depending on the scenario for projections which has been set + It only occurs once on date: scaleup_start_date, + called by initialise_simulation + """ + + def __init__(self, module): + super().__init__(module) + + def apply(self, population): + + self.module.update_parameters_for_program_scaleup() + + class MalariaIPTp(RegularEvent, PopulationScopeEventMixin): """ malaria prophylaxis for pregnant women diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py index 02d860fe52..aa62f3ea8a 100644 --- a/src/tlo/methods/tb.py +++ b/src/tlo/methods/tb.py @@ -9,7 +9,7 @@ import pandas as pd -from tlo import DateOffset, Module, Parameter, Property, Types, logging +from tlo import Date, DateOffset, Module, Parameter, Property, Types, logging from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor from tlo.methods import Metadata, hiv @@ -376,6 +376,19 @@ def __init__(self, name=None, resourcefilepath=None, run_with_checks=False): Types.LIST, "length of inpatient stay for end-of-life TB patients", ), + # ------------------ scale-up parameters for scenario analysis ------------------ # + "do_scaleup": Parameter( + Types.BOOL, + "argument to determine whether scale-up of program will be implemented" + ), + "scaleup_start_year": Parameter( + Types.INT, + "the year when the scale-up starts (it will occur on 1st January of that year)" + ), + "scaleup_parameters": Parameter( + Types.DICT, + "the parameters and values changed in scenario analysis" + ) } def read_parameters(self, data_folder): @@ -413,6 +426,9 @@ def read_parameters(self, data_folder): .tolist() ) + # load parameters for scale-up projections + p["scaleup_parameters"] = workbook["scaleup_parameters"].set_index('parameter')['scaleup_value'].to_dict() + # 2) Get the DALY weights if "HealthBurden" in self.sim.modules.keys(): # HIV-negative @@ -849,6 +865,13 @@ def initialise_simulation(self, sim): sim.schedule_event(TbSelfCureEvent(self), sim.date) sim.schedule_event(TbActiveCasePoll(self), sim.date + DateOffset(years=1)) + # 2) log at the end of the year + # Optional: Schedule the scale-up of programs + if self.parameters["do_scaleup"]: + scaleup_start_date = Date(self.parameters["scaleup_start_year"], 1, 1) + assert scaleup_start_date >= self.sim.start_date, f"Date {scaleup_start_date} is before simulation starts." + sim.schedule_event(TbScaleUpEvent(self), scaleup_start_date) + # 2) log at the end of the year sim.schedule_event(TbLoggingEvent(self), sim.date + DateOffset(years=1)) @@ -861,6 +884,31 @@ def initialise_simulation(self, sim): TbCheckPropertiesEvent(self), sim.date + pd.DateOffset(months=1) ) + def update_parameters_for_program_scaleup(self): + + p = self.parameters + scaled_params = p["scaleup_parameters"] + + if p["do_scaleup"]: + + # scale-up TB program + # use NTP treatment rates + p["rate_testing_active_tb"]["treatment_coverage"] = scaled_params["tb_treatment_coverage"] + + # increase tb treatment success rates + p["prob_tx_success_ds"] = scaled_params["tb_prob_tx_success_ds"] + p["prob_tx_success_mdr"] = scaled_params["tb_prob_tx_success_mdr"] + p["prob_tx_success_0_4"] = scaled_params["tb_prob_tx_success_0_4"] + p["prob_tx_success_5_14"] = scaled_params["tb_prob_tx_success_5_14"] + + # change first-line testing for TB to xpert + p["first_line_test"] = scaled_params["first_line_test"] + p["second_line_test"] = scaled_params["second_line_test"] + + # increase coverage of IPT + p["ipt_coverage"]["coverage_plhiv"] = scaled_params["ipt_coverage_plhiv"] + p["ipt_coverage"]["coverage_paediatric"] = scaled_params["ipt_coverage_paediatric"] + def on_birth(self, mother_id, child_id): """Initialise properties for a newborn individual allocate IPT for child if mother diagnosed with TB @@ -1367,6 +1415,21 @@ def apply(self, population): self.module.relapse_event(population) +class TbScaleUpEvent(Event, PopulationScopeEventMixin): + """ This event exists to change parameters or functions + depending on the scenario for projections which has been set + It only occurs once on date: scaleup_start_date, + called by initialise_simulation + """ + + def __init__(self, module): + super().__init__(module) + + def apply(self, population): + + self.module.update_parameters_for_program_scaleup() + + class TbActiveEvent(RegularEvent, PopulationScopeEventMixin): """ * check for those with dates of active tb onset within last time-period diff --git a/tests/test_htm_scaleup.py b/tests/test_htm_scaleup.py new file mode 100644 index 0000000000..dbb2638c88 --- /dev/null +++ b/tests/test_htm_scaleup.py @@ -0,0 +1,210 @@ +""" Tests for setting up the HIV, TB and malaria scenarios used for projections """ + +import os +from pathlib import Path + +import pandas as pd + +from tlo import Date, Simulation +from tlo.methods import ( + demography, + enhanced_lifestyle, + epi, + healthburden, + healthseekingbehaviour, + healthsystem, + hiv, + malaria, + simplified_births, + symptommanager, + tb, +) + +resourcefilepath = Path(os.path.dirname(__file__)) / "../resources" + +start_date = Date(2010, 1, 1) +scaleup_start_year = 2012 # <-- the scale-up will occur on 1st January of that year +end_date = Date(2013, 1, 1) + + +def get_sim(seed): + """ + register all necessary modules for the tests to run + """ + + sim = Simulation(start_date=start_date, seed=seed) + + # Register the appropriate modules + sim.register( + demography.Demography(resourcefilepath=resourcefilepath), + simplified_births.SimplifiedBirths(resourcefilepath=resourcefilepath), + enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath), + healthsystem.HealthSystem(resourcefilepath=resourcefilepath), + symptommanager.SymptomManager(resourcefilepath=resourcefilepath), + healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=resourcefilepath), + healthburden.HealthBurden(resourcefilepath=resourcefilepath), + epi.Epi(resourcefilepath=resourcefilepath), + hiv.Hiv(resourcefilepath=resourcefilepath), + tb.Tb(resourcefilepath=resourcefilepath), + malaria.Malaria(resourcefilepath=resourcefilepath), + ) + + return sim + + +def check_initial_params(sim): + + original_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name='parameters') + + # check initial parameters + assert sim.modules["Hiv"].parameters["beta"] == \ + original_params.loc[original_params.parameter_name == "beta", "value"].values[0] + assert sim.modules["Hiv"].parameters["prob_prep_for_fsw_after_hiv_test"] == original_params.loc[ + original_params.parameter_name == "prob_prep_for_fsw_after_hiv_test", "value"].values[0] + assert sim.modules["Hiv"].parameters["prob_prep_for_agyw"] == original_params.loc[ + original_params.parameter_name == "prob_prep_for_agyw", "value"].values[0] + assert sim.modules["Hiv"].parameters["probability_of_being_retained_on_prep_every_3_months"] == original_params.loc[ + original_params.parameter_name == "probability_of_being_retained_on_prep_every_3_months", "value"].values[0] + assert sim.modules["Hiv"].parameters["prob_circ_after_hiv_test"] == original_params.loc[ + original_params.parameter_name == "prob_circ_after_hiv_test", "value"].values[0] + + +def test_hiv_scale_up(seed): + """ test hiv program scale-up changes parameters correctly + and on correct date """ + + original_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name="parameters") + new_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name="scaleup_parameters") + + popsize = 100 + + sim = get_sim(seed=seed) + + # check initial parameters + check_initial_params(sim) + + # update parameters to instruct there to be a scale-up + sim.modules["Hiv"].parameters["do_scaleup"] = True + sim.modules["Hiv"].parameters["scaleup_start_year"] = scaleup_start_year + + # Make the population + sim.make_initial_population(n=popsize) + sim.simulate(end_date=end_date) + + # check HIV parameters changed + assert sim.modules["Hiv"].parameters["beta"] < original_params.loc[ + original_params.parameter_name == "beta", "value"].values[0] + assert sim.modules["Hiv"].parameters["prob_prep_for_fsw_after_hiv_test"] == new_params.loc[ + new_params.parameter == "prob_prep_for_fsw_after_hiv_test", "scaleup_value"].values[0] + assert sim.modules["Hiv"].parameters["prob_prep_for_agyw"] == new_params.loc[ + new_params.parameter == "prob_prep_for_agyw", "scaleup_value"].values[0] + assert sim.modules["Hiv"].parameters["probability_of_being_retained_on_prep_every_3_months"] == new_params.loc[ + new_params.parameter == "probability_of_being_retained_on_prep_every_3_months", "scaleup_value"].values[0] + assert sim.modules["Hiv"].parameters["prob_circ_after_hiv_test"] == new_params.loc[ + new_params.parameter == "prob_circ_after_hiv_test", "scaleup_value"].values[0] + + # check malaria parameters unchanged + mal_original_params = pd.read_excel(resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx', + sheet_name="parameters") + mal_rdt_testing = pd.read_excel(resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx', + sheet_name="WHO_TestData2023") + + assert sim.modules["Malaria"].parameters["prob_malaria_case_tests"] == mal_original_params.loc[ + mal_original_params.parameter_name == "prob_malaria_case_tests", "value"].values[0] + pd.testing.assert_series_equal(sim.modules["Malaria"].parameters["rdt_testing_rates"]["Rate_rdt_testing"], + mal_rdt_testing["Rate_rdt_testing"]) + + # all irs coverage levels should be < 1.0 + assert sim.modules["Malaria"].itn_irs['irs_rate'].all() < 1.0 + # itn rates for 2019 onwards + assert sim.modules["Malaria"].parameters["itn"] == mal_original_params.loc[ + mal_original_params.parameter_name == "itn", "value"].values[0] + + # check tb parameters unchanged + tb_original_params = pd.read_excel(resourcefilepath / 'ResourceFile_TB.xlsx', sheet_name="parameters") + tb_testing = pd.read_excel(resourcefilepath / 'ResourceFile_TB.xlsx', sheet_name="NTP2019") + + pd.testing.assert_series_equal(sim.modules["Tb"].parameters["rate_testing_active_tb"]["treatment_coverage"], + tb_testing["treatment_coverage"]) + assert sim.modules["Tb"].parameters["prob_tx_success_ds"] == tb_original_params.loc[ + tb_original_params.parameter_name == "prob_tx_success_ds", "value"].values[0] + assert sim.modules["Tb"].parameters["prob_tx_success_mdr"] == tb_original_params.loc[ + tb_original_params.parameter_name == "prob_tx_success_mdr", "value"].values[0] + assert sim.modules["Tb"].parameters["prob_tx_success_0_4"] == tb_original_params.loc[ + tb_original_params.parameter_name == "prob_tx_success_0_4", "value"].values[0] + assert sim.modules["Tb"].parameters["prob_tx_success_5_14"] == tb_original_params.loc[ + tb_original_params.parameter_name == "prob_tx_success_5_14", "value"].values[0] + assert sim.modules["Tb"].parameters["first_line_test"] == tb_original_params.loc[ + tb_original_params.parameter_name == "first_line_test", "value"].values[0] + + +def test_htm_scale_up(seed): + """ test hiv/tb/malaria program scale-up changes parameters correctly + and on correct date """ + + # Load data on HIV prevalence + original_hiv_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name="parameters") + new_hiv_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name="scaleup_parameters") + + popsize = 100 + + sim = get_sim(seed=seed) + + # check initial parameters + check_initial_params(sim) + + # update parameters + sim.modules["Hiv"].parameters["do_scaleup"] = True + sim.modules["Hiv"].parameters["scaleup_start_year"] = scaleup_start_year + sim.modules["Tb"].parameters["do_scaleup"] = True + sim.modules["Tb"].parameters["scaleup_start_year"] = scaleup_start_year + sim.modules["Malaria"].parameters["do_scaleup"] = True + sim.modules["Malaria"].parameters["scaleup_start_year"] = scaleup_start_year + + # Make the population + sim.make_initial_population(n=popsize) + sim.simulate(end_date=end_date) + + # check HIV parameters changed + assert sim.modules["Hiv"].parameters["beta"] < original_hiv_params.loc[ + original_hiv_params.parameter_name == "beta", "value"].values[0] + assert sim.modules["Hiv"].parameters["prob_prep_for_fsw_after_hiv_test"] == new_hiv_params.loc[ + new_hiv_params.parameter == "prob_prep_for_fsw_after_hiv_test", "scaleup_value"].values[0] + assert sim.modules["Hiv"].parameters["prob_prep_for_agyw"] == new_hiv_params.loc[ + new_hiv_params.parameter == "prob_prep_for_agyw", "scaleup_value"].values[0] + assert sim.modules["Hiv"].parameters["probability_of_being_retained_on_prep_every_3_months"] == new_hiv_params.loc[ + new_hiv_params.parameter == "probability_of_being_retained_on_prep_every_3_months", "scaleup_value"].values[0] + assert sim.modules["Hiv"].parameters["prob_circ_after_hiv_test"] == new_hiv_params.loc[ + new_hiv_params.parameter == "prob_circ_after_hiv_test", "scaleup_value"].values[0] + + # check malaria parameters changed + new_mal_params = pd.read_excel(resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx', + sheet_name="scaleup_parameters") + + assert sim.modules["Malaria"].parameters["prob_malaria_case_tests"] == new_mal_params.loc[ + new_mal_params.parameter == "prob_malaria_case_tests", "scaleup_value"].values[0] + assert sim.modules["Malaria"].parameters["rdt_testing_rates"]["Rate_rdt_testing"].eq(new_mal_params.loc[ + new_mal_params.parameter == "rdt_testing_rates", "scaleup_value"].values[0]).all() + + # some irs coverage levels should now = 1.0 + assert sim.modules["Malaria"].itn_irs['irs_rate'].any() == 1.0 + # itn rates for 2019 onwards + assert sim.modules["Malaria"].parameters["itn"] == new_mal_params.loc[ + new_mal_params.parameter == "itn", "scaleup_value"].values[0] + + # check tb parameters changed + new_tb_params = pd.read_excel(resourcefilepath / 'ResourceFile_TB.xlsx', sheet_name="scaleup_parameters") + + assert sim.modules["Tb"].parameters["rate_testing_active_tb"]["treatment_coverage"].eq(new_tb_params.loc[ + new_tb_params.parameter == "tb_treatment_coverage", "scaleup_value"].values[0]).all() + assert sim.modules["Tb"].parameters["prob_tx_success_ds"] == new_tb_params.loc[ + new_tb_params.parameter == "tb_prob_tx_success_ds", "scaleup_value"].values[0] + assert sim.modules["Tb"].parameters["prob_tx_success_mdr"] == new_tb_params.loc[ + new_tb_params.parameter == "tb_prob_tx_success_mdr", "scaleup_value"].values[0] + assert sim.modules["Tb"].parameters["prob_tx_success_0_4"] == new_tb_params.loc[ + new_tb_params.parameter == "tb_prob_tx_success_0_4", "scaleup_value"].values[0] + assert sim.modules["Tb"].parameters["prob_tx_success_5_14"] == new_tb_params.loc[ + new_tb_params.parameter == "tb_prob_tx_success_5_14", "scaleup_value"].values[0] + assert sim.modules["Tb"].parameters["first_line_test"] == new_tb_params.loc[ + new_tb_params.parameter == "first_line_test", "scaleup_value"].values[0] + From e9263803787825168955b0e24ed15c383a2ba0dd Mon Sep 17 00:00:00 2001 From: Matt Graham Date: Sun, 30 Jun 2024 22:15:08 +0100 Subject: [PATCH 06/19] Update ruff check command to fix failing checks (#1407) * Update ruff check command * Fix type comparison with equality operator --- src/tlo/core.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tlo/core.py b/src/tlo/core.py index fe92203e56..3d3fd3c171 100644 --- a/src/tlo/core.py +++ b/src/tlo/core.py @@ -288,7 +288,7 @@ def load_parameters_from_dataframe(self, resource: pd.DataFrame): f"The value of '{parameter_value}' for parameter '{parameter_name}' " f"could not be parsed as a {parameter_definition.type_.name} data type" ) - if parameter_definition.python_type == list: + if parameter_definition.python_type is list: try: # chose json.loads instead of save_eval # because it raises error instead of joining two strings without a comma diff --git a/tox.ini b/tox.ini index 94949bd6d8..e2417422e8 100644 --- a/tox.ini +++ b/tox.ini @@ -93,7 +93,7 @@ commands = twine check dist/*.tar.gz dist/*.whl ; ignore that _version.py file generated by setuptools_scm is not tracked by VCS check-manifest --ignore **/_version.py {toxinidir} - ruff src tests + ruff check src tests isort --check-only --diff src tests pylint src tests python {toxinidir}/src/scripts/automation/update_citation.py --check From 5920f513192cf2f7310a9c05f826aa151f85e7e8 Mon Sep 17 00:00:00 2001 From: Watipaso Mulwafu <39279950+thewati@users.noreply.github.com> Date: Wed, 10 Jul 2024 08:23:05 +0200 Subject: [PATCH 07/19] Fixing recurrence of HSIs in RTI (#1408) --- resources/ResourceFile_RTI.xlsx | 4 +- src/tlo/methods/rti.py | 75 ++++++++++++++++++++++++++++----- 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/resources/ResourceFile_RTI.xlsx b/resources/ResourceFile_RTI.xlsx index 68cdd18422..553d6febb0 100644 --- a/resources/ResourceFile_RTI.xlsx +++ b/resources/ResourceFile_RTI.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2c11ada2e8b77675950b61fc8e0efd1c4fa35dffaecaf1029eafd61892a7cefb -size 13949 +oid sha256:d950c5d769848fb226db8c1a7d7796c8e43cc2590806f846b98a4bbef6840948 +size 13776 diff --git a/src/tlo/methods/rti.py b/src/tlo/methods/rti.py index b76fb40e9f..13ddee6a86 100644 --- a/src/tlo/methods/rti.py +++ b/src/tlo/methods/rti.py @@ -1016,6 +1016,10 @@ def __init__(self, name=None, resourcefilepath=None): Types.INT, "A cut-off score above which an injuries will be considered severe enough to cause mortality in those who" "have not sought care." + ), + 'maximum_number_of_times_HSI_events_should_run': Parameter( + Types.INT, + "limit on the number of times an HSI event can run" ) } @@ -3825,9 +3829,12 @@ def __init__(self, module, person_id): self.TREATMENT_ID = 'Rti_ShockTreatment' self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({'AccidentsandEmerg': 1}) self.ACCEPTED_FACILITY_LEVEL = '1b' + self._number_of_times_this_event_has_run = 0 + self._maximum_number_times_event_should_run = self.module.parameters['maximum_number_of_times_HSI_events_should_run'] def apply(self, person_id, squeeze_factor): df = self.sim.population.props + self._number_of_times_this_event_has_run += 1 # determine if this is a child if df.loc[person_id, 'age_years'] < 15: is_child = True @@ -3865,7 +3872,8 @@ def apply(self, person_id, squeeze_factor): df.at[person_id, 'rt_in_shock'] = False self.add_equipment({'Infusion pump', 'Drip stand', 'Oxygen cylinder, with regulator', 'Nasal Prongs'}) else: - self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) + if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run: + self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) return self.make_appt_footprint({}) def did_not_run(self): @@ -3918,11 +3926,15 @@ def __init__(self, module, person_id): self.TREATMENT_ID = 'Rti_FractureCast' self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({'AccidentsandEmerg': 1}) self.ACCEPTED_FACILITY_LEVEL = '1b' + self._number_of_times_this_event_has_run = 0 + self._maximum_number_times_event_should_run = self.module.parameters[ + 'maximum_number_of_times_HSI_events_should_run'] def apply(self, person_id, squeeze_factor): # Get the population and health system df = self.sim.population.props p = df.loc[person_id] + self._number_of_times_this_event_has_run += 1 # if the person isn't alive return a blank footprint if not df.at[person_id, 'is_alive']: return self.make_appt_footprint({}) @@ -4017,7 +4029,8 @@ def apply(self, person_id, squeeze_factor): df.loc[person_id, 'rt_injuries_to_cast'].clear() df.loc[person_id, 'rt_date_death_no_med'] = pd.NaT else: - self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) + if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run: + self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) if pd.isnull(df.loc[person_id, 'rt_date_death_no_med']): df.loc[person_id, 'rt_date_death_no_med'] = self.sim.date + DateOffset(days=7) logger.debug(key='rti_general_message', @@ -4057,9 +4070,13 @@ def __init__(self, module, person_id): self.TREATMENT_ID = 'Rti_OpenFractureTreatment' self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({'MinorSurg': 1}) self.ACCEPTED_FACILITY_LEVEL = '1b' + self._number_of_times_this_event_has_run = 0 + self._maximum_number_times_event_should_run = self.module.parameters[ + 'maximum_number_of_times_HSI_events_should_run'] def apply(self, person_id, squeeze_factor): df = self.sim.population.props + self._number_of_times_this_event_has_run += 1 if not df.at[person_id, 'is_alive']: return self.make_appt_footprint({}) road_traffic_injuries = self.sim.modules['RTI'] @@ -4131,7 +4148,8 @@ def apply(self, person_id, squeeze_factor): if code[0] in df.loc[person_id, 'rt_injuries_for_open_fracture_treatment']: df.loc[person_id, 'rt_injuries_for_open_fracture_treatment'].remove(code[0]) else: - self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) + if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run: + self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) if pd.isnull(df.loc[person_id, 'rt_date_death_no_med']): df.loc[person_id, 'rt_date_death_no_med'] = self.sim.date + DateOffset(days=7) logger.debug(key='rti_general_message', @@ -4174,10 +4192,15 @@ def __init__(self, module, person_id): self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({ ('Under5OPD' if self.sim.population.props.at[person_id, "age_years"] < 5 else 'Over5OPD'): 1}) self.ACCEPTED_FACILITY_LEVEL = '1b' + self._number_of_times_this_event_has_run = 0 + self._maximum_number_times_event_should_run = self.module.parameters[ + 'maximum_number_of_times_HSI_events_should_run'] def apply(self, person_id, squeeze_factor): get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name df = self.sim.population.props + self._number_of_times_this_event_has_run += 1 + if not df.at[person_id, 'is_alive']: return self.make_appt_footprint({}) road_traffic_injuries = self.sim.modules['RTI'] @@ -4222,7 +4245,8 @@ def apply(self, person_id, squeeze_factor): assert df.loc[person_id, date_to_remove_daly_column] > self.sim.date df.loc[person_id, 'rt_date_death_no_med'] = pd.NaT else: - self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) + if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run: + self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) if pd.isnull(df.loc[person_id, 'rt_date_death_no_med']): df.loc[person_id, 'rt_date_death_no_med'] = self.sim.date + DateOffset(days=7) logger.debug(key='rti_general_message', @@ -4269,11 +4293,15 @@ def __init__(self, module, person_id): p = self.module.parameters self.prob_mild_burns = p['prob_mild_burns'] + self._number_of_times_this_event_has_run = 0 + self._maximum_number_times_event_should_run = p['maximum_number_of_times_HSI_events_should_run'] def apply(self, person_id, squeeze_factor): get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name df = self.sim.population.props + self._number_of_times_this_event_has_run += 1 + if not df.at[person_id, 'is_alive']: return self.make_appt_footprint({}) road_traffic_injuries = self.sim.modules['RTI'] @@ -4346,7 +4374,8 @@ def apply(self, person_id, squeeze_factor): ) df.loc[person_id, 'rt_date_death_no_med'] = pd.NaT else: - self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) + if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run: + self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) if pd.isnull(df.loc[person_id, 'rt_date_death_no_med']): df.loc[person_id, 'rt_date_death_no_med'] = self.sim.date + DateOffset(days=7) logger.debug(key='rti_general_message', @@ -4373,9 +4402,14 @@ def __init__(self, module, person_id): self.TREATMENT_ID = 'Rti_TetanusVaccine' self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({'EPI': 1}) self.ACCEPTED_FACILITY_LEVEL = '1b' + self._number_of_times_this_event_has_run = 0 + self._maximum_number_times_event_should_run = self.module.parameters[ + 'maximum_number_of_times_HSI_events_should_run'] def apply(self, person_id, squeeze_factor): df = self.sim.population.props + self._number_of_times_this_event_has_run += 1 + if not df.at[person_id, 'is_alive']: return self.make_appt_footprint({}) person_injuries = df.loc[[person_id], RTI.INJURY_COLUMNS] @@ -4404,7 +4438,8 @@ def apply(self, person_id, squeeze_factor): logger.debug(key='rti_general_message', data=f"Tetanus vaccine requested for person {person_id} and given") else: - self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) + if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run: + self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) logger.debug(key='rti_general_message', data=f"Tetanus vaccine requested for person {person_id}, not given") return self.make_appt_footprint({}) @@ -4434,9 +4469,14 @@ def __init__(self, module, person_id): self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({ ('Under5OPD' if self.sim.population.props.at[person_id, "age_years"] < 5 else 'Over5OPD'): 1}) self.ACCEPTED_FACILITY_LEVEL = '1b' + self._number_of_times_this_event_has_run = 0 + self._maximum_number_times_event_should_run = self.module.parameters[ + 'maximum_number_of_times_HSI_events_should_run'] def apply(self, person_id, squeeze_factor): df = self.sim.population.props + self._number_of_times_this_event_has_run += 1 + if not df.at[person_id, 'is_alive']: return self.make_appt_footprint({}) # Check that the person sent here is alive, has been through A&E and RTI_Med_int @@ -4545,7 +4585,8 @@ def apply(self, person_id, squeeze_factor): data=dict_to_output, description='Pain medicine successfully provided to the person') else: - self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) + if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run: + self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) logger.debug(key='rti_general_message', data=f"This facility has no pain management available for their mild pain, person " f"{person_id}.") @@ -4576,7 +4617,8 @@ def apply(self, person_id, squeeze_factor): data=dict_to_output, description='Pain medicine successfully provided to the person') else: - self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) + if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run: + self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) logger.debug(key='rti_general_message', data=f"This facility has no pain management available for moderate pain for person " f"{person_id}.") @@ -4608,7 +4650,8 @@ def apply(self, person_id, squeeze_factor): data=dict_to_output, description='Pain medicine successfully provided to the person') else: - self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) + if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run: + self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) logger.debug(key='rti_general_message', data=f"This facility has no pain management available for severe pain for person " f"{person_id}.") @@ -4736,6 +4779,8 @@ def __init__(self, module, person_id): self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({'MajorSurg': 1}) self.ACCEPTED_FACILITY_LEVEL = '1b' self.BEDDAYS_FOOTPRINT = self.make_beddays_footprint({}) + self._number_of_times_this_event_has_run = 0 + self._maximum_number_times_event_should_run = self.module.parameters['maximum_number_of_times_HSI_events_should_run'] p = self.module.parameters self.prob_perm_disability_with_treatment_severe_TBI = p['prob_perm_disability_with_treatment_severe_TBI'] @@ -4743,6 +4788,7 @@ def __init__(self, module, person_id): self.treated_code = 'none' def apply(self, person_id, squeeze_factor): + self._number_of_times_this_event_has_run += 1 df = self.sim.population.props rng = self.module.rng road_traffic_injuries = self.sim.modules['RTI'] @@ -5015,7 +5061,8 @@ def apply(self, person_id, squeeze_factor): ['Treated injury code not removed', self.treated_code] df.loc[person_id, 'rt_date_death_no_med'] = pd.NaT else: - self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) + if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run: + self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) if pd.isnull(df.loc[person_id, 'rt_date_death_no_med']): df.loc[person_id, 'rt_date_death_no_med'] = self.sim.date + DateOffset(days=7) return self.make_appt_footprint({}) @@ -5081,7 +5128,12 @@ def __init__(self, module, person_id): self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({'MinorSurg': 1}) self.ACCEPTED_FACILITY_LEVEL = '1b' + self._number_of_times_this_event_has_run = 0 + self._maximum_number_times_event_should_run = self.module.parameters[ + 'maximum_number_of_times_HSI_events_should_run'] + def apply(self, person_id, squeeze_factor): + self._number_of_times_this_event_has_run += 1 df = self.sim.population.props if not df.at[person_id, 'is_alive']: return self.make_appt_footprint({}) @@ -5202,7 +5254,8 @@ def apply(self, person_id, squeeze_factor): ['Injury treated not removed', treated_code] df.loc[person_id, 'rt_date_death_no_med'] = pd.NaT else: - self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) + if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run: + self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self) if pd.isnull(df.loc[person_id, 'rt_date_death_no_med']): df.loc[person_id, 'rt_date_death_no_med'] = self.sim.date + DateOffset(days=7) logger.debug(key='rti_general_message', From 2dff6751ab1294ef42917880dc1ddc46b167e522 Mon Sep 17 00:00:00 2001 From: Will Graham <32364977+willGraham01@users.noreply.github.com> Date: Wed, 17 Jul 2024 09:46:35 +0100 Subject: [PATCH 08/19] Update run-profiling.yaml with note about new token (#1424) * Update run-profiling.yaml with note about new token * Update run-profiling.yaml --- .github/workflows/run-profiling.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-profiling.yaml b/.github/workflows/run-profiling.yaml index af0611b074..a28acda1b0 100644 --- a/.github/workflows/run-profiling.yaml +++ b/.github/workflows/run-profiling.yaml @@ -163,7 +163,7 @@ jobs: ## The token provided needs contents and pages access to the target repo ## Token can be (re)generated by a member of the UCL organisation, ## the current member is the rc-softdev-admin. - ## [10-07-2023] The current token will expire 10-07-2024 + ## [17-07-2024] New token generated, will expire 10-07-2025 - name: Push results to profiling repository uses: dmnemec/copy_file_to_another_repo_action@v1.1.1 env: From 439b3ba84abd7047b4268bed0d8d35d53118bb92 Mon Sep 17 00:00:00 2001 From: Will Graham <32364977+willGraham01@users.noreply.github.com> Date: Mon, 22 Jul 2024 09:15:19 +0100 Subject: [PATCH 09/19] `BedDays` availability switch now correctly updates capacities (#1352) --- src/tlo/methods/bed_days.py | 38 +++++++++++++++- src/tlo/methods/healthsystem.py | 6 ++- tests/test_beddays.py | 79 +++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 3 deletions(-) diff --git a/src/tlo/methods/bed_days.py b/src/tlo/methods/bed_days.py index ef501f3b2e..7adb6de60c 100644 --- a/src/tlo/methods/bed_days.py +++ b/src/tlo/methods/bed_days.py @@ -5,12 +5,12 @@ """ from collections import defaultdict -from typing import Dict, Tuple +from typing import Dict, Literal, Tuple import numpy as np import pandas as pd -from tlo import Property, Types, logging +from tlo import Date, Property, Types, logging # --------------------------------------------------------------------------------------------------------- # CLASS DEFINITIONS @@ -145,6 +145,40 @@ def initialise_beddays_tracker(self, model_to_data_popsize_ratio=1.0): assert not df.isna().any().any() self.bed_tracker[bed_type] = df + def switch_beddays_availability( + self, + new_availability: Literal["all", "none", "default"], + effective_on_and_from: Date, + model_to_data_popsize_ratio: float = 1.0, + ) -> None: + """ + Action to be taken if the beddays availability changes in the middle + of the simulation. + + If bed capacities are reduced below the currently scheduled occupancy, + inpatients are not evicted from beds and are allowed to remain in the + bed until they are scheduled to leave. Obviously, no new patients will + be admitted if there is no room in the new capacities. + + :param new_availability: The new bed availability. See __init__ for details. + :param effective_on_and_from: First day from which the new capacities will be imposed. + :param model_to_data_popsize_ratio: As in initialise_population. + """ + # Store new bed availability + self.availability = new_availability + # Before we update the bed capacity, we need to store its old values + # This is because we will need to update the trackers to reflect the new# + # maximum capacities for each bed type. + old_max_capacities: pd.DataFrame = self._scaled_capacity.copy() + # Set the new capacity for beds + self.set_scaled_capacity(model_to_data_popsize_ratio) + # Compute the difference between the new max capacities and the old max capacities + difference_in_max = self._scaled_capacity - old_max_capacities + # For each tracker, after the effective date, impose the difference on the max + # number of beds + for bed_type, tracker in self.bed_tracker.items(): + tracker.loc[effective_on_and_from:] += difference_in_max[bed_type] + def on_start_of_day(self): """Things to do at the start of each new day: * Refresh inpatient status diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index 8099346ddf..db9173d15b 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -2820,7 +2820,11 @@ def apply(self, population): self.module.consumables.availability = self._parameters['cons_availability'] if 'beds_availability' in self._parameters: - self.module.bed_days.availability = self._parameters['beds_availability'] + self.module.bed_days.switch_beddays_availability( + new_availability=self._parameters["beds_availability"], + effective_on_and_from=self.sim.date, + model_to_data_popsize_ratio=self.sim.modules["Demography"].initial_model_to_data_popsize_ratio + ) if 'equip_availability' in self._parameters: self.module.equipment.availability = self._parameters['equip_availability'] diff --git a/tests/test_beddays.py b/tests/test_beddays.py index 614719fc86..f3f3e7f087 100644 --- a/tests/test_beddays.py +++ b/tests/test_beddays.py @@ -973,3 +973,82 @@ def apply(self, person_id, squeeze_factor): # Check that the facility_id is included for each entry in the `HSI_Events` log, including HSI Events for # in-patient appointments. assert not (log_hsi['Facility_ID'] == -99).any() + +def test_beddays_availability_switch(seed): + """ + Test that calling bed_days.switch_beddays_availability correctly updates the + bed capacities and adjusts the existing trackers to reflect the new capacities. + """ + sim = Simulation(start_date=start_date, seed=seed) + sim.register( + demography.Demography(resourcefilepath=resourcefilepath), + healthsystem.HealthSystem(resourcefilepath=resourcefilepath), + ) + + # get shortcut to HealthSystem Module + hs: healthsystem.HealthSystem = sim.modules["HealthSystem"] + + # As obtained from the resource file + facility_id_with_patient = 128 + facility_id_without_patient = 129 + bedtype1_init_capacity = 5 + bedtype2_init_capacity = 10 + + # Create a simple bed capacity dataframe with capacity designated for two regions + hs.parameters["BedCapacity"] = pd.DataFrame( + data={ + "Facility_ID": [ + facility_id_with_patient, #<-- patient 0 is admitted here + facility_id_without_patient, + ], + "bedtype1": bedtype1_init_capacity, + "bedtype2": bedtype2_init_capacity, + } + ) + sim.make_initial_population(n=100) + sim.simulate(end_date=start_date) + + day_2 = start_date + pd.DateOffset(days=1) + day_3 = start_date + pd.DateOffset(days=2) + day_4 = start_date + pd.DateOffset(days=3) + + bed_days = hs.bed_days + # Reset the bed occupancies + bed_days.initialise_beddays_tracker() + # Have a patient occupy a bed at the start of the simulation + bed_days.impose_beddays_footprint(person_id=0, footprint={"bedtype1": 3, "bedtype2": 0}) + + # Have the bed_days availability switch to "none" on the 2nd simulation day + bed_days.switch_beddays_availability("none", effective_on_and_from=day_2) + + # We should now see that the scaled capacities are all zero + assert ( + not bed_days._scaled_capacity.any().any() + ), "At least one bed capacity was not set to 0" + # We should also see that bedtype1 should have -1 beds available for days 2 and 3 of the simulation, + # due to the existing occupancy and the new capacity of 0. + # It should have 4 beds available on the first day (since the original capacity was 5 and the availability + # switch happens day 2). + # It should then have 0 beds available after (not including) day 3 + bedtype1: pd.DataFrame = bed_days.bed_tracker["bedtype1"] + bedtype2: pd.DataFrame = bed_days.bed_tracker["bedtype2"] + + assert ( + bedtype1.loc[start_date, facility_id_with_patient] == bedtype1_init_capacity - 1 + and bedtype1.loc[start_date, facility_id_without_patient] + == bedtype1_init_capacity + ), "Day 1 capacities were incorrectly affected" + assert (bedtype1.loc[day_2:day_3, facility_id_with_patient] == -1).all() and ( + bedtype1.loc[day_2:day_3, facility_id_without_patient] == 0 + ).all(), "Day 2 & 3 capacities were not updated correctly" + assert ( + (bedtype1.loc[day_4:, :] == 0).all().all() + ), "Day 4 onwards did not have correct capacity" + + # Bedtype 2 should have also have been updated, but there is no funny business here. + assert ( + (bedtype2.loc[day_2:, :] == 0).all().all() + ), "Bedtype 2 was not updated correctly" + assert ( + (bedtype2.loc[start_date, :] == bedtype2_init_capacity).all().all() + ), "Bedtype 2 had capacity updated on the incorrect dates" From 49eaf108d60b7ca51071e7d0838ef4f9fbe39615 Mon Sep 17 00:00:00 2001 From: Will Graham <32364977+willGraham01@users.noreply.github.com> Date: Tue, 23 Jul 2024 10:15:49 +0100 Subject: [PATCH 10/19] Use `IndividualDetails` context to speed up `SymptomManager.has_what` (#1423) * Use the individualdetails context for symptommanager.has_what if its available * Update has_what test to include new usage case * Update docstring and function call signature * Use keyword syntax for has_what across the codebase now * Revise method docstring and remove un-necessary param * Fix tests that seem to use the disease_module keyword * Fix missed test in symptommanager that needs KWARGS * Write test for new has_what functionality --- src/tlo/methods/alri.py | 8 +- src/tlo/methods/bladder_cancer.py | 4 +- src/tlo/methods/breast_cancer.py | 2 +- src/tlo/methods/depression.py | 2 +- src/tlo/methods/hsi_generic_first_appts.py | 6 +- src/tlo/methods/malaria.py | 8 +- src/tlo/methods/measles.py | 2 +- src/tlo/methods/oesophagealcancer.py | 2 +- src/tlo/methods/other_adult_cancers.py | 2 +- src/tlo/methods/prostate_cancer.py | 4 +- src/tlo/methods/symptommanager.py | 95 +++++++++++--- src/tlo/methods/tb.py | 6 +- tests/test_alri.py | 12 +- tests/test_cardiometabolicdisorders.py | 2 +- tests/test_copd.py | 16 ++- tests/test_hiv.py | 8 +- tests/test_malaria.py | 8 +- tests/test_symptommanager.py | 145 +++++++++++++++++++-- tests/test_tb.py | 8 +- 19 files changed, 264 insertions(+), 76 deletions(-) diff --git a/src/tlo/methods/alri.py b/src/tlo/methods/alri.py index c27a54dd30..70ac14fe2d 100644 --- a/src/tlo/methods/alri.py +++ b/src/tlo/methods/alri.py @@ -1253,7 +1253,7 @@ def do_effects_of_treatment_and_return_outcome(self, person_id, antibiotic_provi # Gather underlying properties that will affect success of treatment SpO2_level = person.ri_SpO2_level - symptoms = self.sim.modules['SymptomManager'].has_what(person_id) + symptoms = self.sim.modules['SymptomManager'].has_what(person_id=person_id) imci_symptom_based_classification = self.get_imci_classification_based_on_symptoms( child_is_younger_than_2_months=person.age_exact_years < (2.0 / 12.0), symptoms=symptoms, @@ -2726,7 +2726,7 @@ def apply(self, person_id, squeeze_factor): return # Do nothing if the persons does not have indicating symptoms - symptoms = self.sim.modules['SymptomManager'].has_what(person_id) + symptoms = self.sim.modules['SymptomManager'].has_what(person_id=person_id) if not {'cough', 'difficult_breathing'}.intersection(symptoms): return self.make_appt_footprint({}) @@ -3009,7 +3009,7 @@ def apply(self, person_id): assert 'danger_signs_pneumonia' == self.module.get_imci_classification_based_on_symptoms( child_is_younger_than_2_months=df.at[person_id, 'age_exact_years'] < (2.0 / 12.0), - symptoms=self.sim.modules['SymptomManager'].has_what(person_id) + symptoms=self.sim.modules['SymptomManager'].has_what(person_id=person_id) ) @@ -3040,7 +3040,7 @@ def apply(self, person_id): assert 'fast_breathing_pneumonia' == \ self.module.get_imci_classification_based_on_symptoms( - child_is_younger_than_2_months=False, symptoms=self.sim.modules['SymptomManager'].has_what(person_id) + child_is_younger_than_2_months=False, symptoms=self.sim.modules['SymptomManager'].has_what(person_id=person_id) ) diff --git a/src/tlo/methods/bladder_cancer.py b/src/tlo/methods/bladder_cancer.py index 113d19fde2..52271f6f16 100644 --- a/src/tlo/methods/bladder_cancer.py +++ b/src/tlo/methods/bladder_cancer.py @@ -718,7 +718,7 @@ def apply(self, person_id, squeeze_factor): return hs.get_blank_appt_footprint() # Check that this event has been called for someone with the symptom blood_urine - assert 'blood_urine' in self.sim.modules['SymptomManager'].has_what(person_id) + assert 'blood_urine' in self.sim.modules['SymptomManager'].has_what(person_id=person_id) # If the person is already diagnosed, then take no action: if not pd.isnull(df.at[person_id, "bc_date_diagnosis"]): @@ -791,7 +791,7 @@ def apply(self, person_id, squeeze_factor): return hs.get_blank_appt_footprint() # Check that this event has been called for someone with the symptom pelvic_pain - assert 'pelvic_pain' in self.sim.modules['SymptomManager'].has_what(person_id) + assert 'pelvic_pain' in self.sim.modules['SymptomManager'].has_what(person_id=person_id) # If the person is already diagnosed, then take no action: if not pd.isnull(df.at[person_id, "bc_date_diagnosis"]): diff --git a/src/tlo/methods/breast_cancer.py b/src/tlo/methods/breast_cancer.py index d362f7ce08..a55c6f4930 100644 --- a/src/tlo/methods/breast_cancer.py +++ b/src/tlo/methods/breast_cancer.py @@ -685,7 +685,7 @@ def apply(self, person_id, squeeze_factor): return hs.get_blank_appt_footprint() # Check that this event has been called for someone with the symptom breast_lump_discernible - assert 'breast_lump_discernible' in self.sim.modules['SymptomManager'].has_what(person_id) + assert 'breast_lump_discernible' in self.sim.modules['SymptomManager'].has_what(person_id=person_id) # If the person is already diagnosed, then take no action: if not pd.isnull(df.at[person_id, "brc_date_diagnosis"]): diff --git a/src/tlo/methods/depression.py b/src/tlo/methods/depression.py index 81ae29403e..4e6825cbc5 100644 --- a/src/tlo/methods/depression.py +++ b/src/tlo/methods/depression.py @@ -593,7 +593,7 @@ def do_on_presentation_to_care(self, person_id: int, hsi_event: HSI_Event): and there may need to be screening for depression. """ if self._check_for_suspected_depression( - self.sim.modules["SymptomManager"].has_what(person_id), + self.sim.modules["SymptomManager"].has_what(person_id=person_id), hsi_event.TREATMENT_ID, self.sim.population.props.at[person_id, "de_ever_diagnosed_depression"], ): diff --git a/src/tlo/methods/hsi_generic_first_appts.py b/src/tlo/methods/hsi_generic_first_appts.py index 30f4d40ac7..37f6c5e261 100644 --- a/src/tlo/methods/hsi_generic_first_appts.py +++ b/src/tlo/methods/hsi_generic_first_appts.py @@ -184,8 +184,10 @@ def apply(self, person_id: int, squeeze_factor: float = 0.0) -> None: if not individual_properties["is_alive"]: return # Pre-evaluate symptoms for individual to avoid repeat accesses - # TODO: Use individual_properties to populate symptoms - symptoms = self.sim.modules["SymptomManager"].has_what(self.target) + # Use the individual_properties context here to save independent DF lookups + symptoms = self.sim.modules["SymptomManager"].has_what( + individual_details=individual_properties + ) schedule_hsi_event = self.sim.modules["HealthSystem"].schedule_hsi_event for module in self.sim.modules.values(): if isinstance(module, GenericFirstAppointmentsMixin): diff --git a/src/tlo/methods/malaria.py b/src/tlo/methods/malaria.py index f322783717..3e273245eb 100644 --- a/src/tlo/methods/malaria.py +++ b/src/tlo/methods/malaria.py @@ -1060,7 +1060,7 @@ def apply(self, person_id, squeeze_factor): ) # Log the test: line-list of summary information about each test - fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id) + fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id=person_id) person_details_for_test = { 'person_id': person_id, 'age': df.at[person_id, 'age_years'], @@ -1152,7 +1152,7 @@ def apply(self, person_id, squeeze_factor): ) # Log the test: line-list of summary information about each test - fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id) + fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id=person_id) person_details_for_test = { 'person_id': person_id, 'age': df.at[person_id, 'age_years'], @@ -1214,7 +1214,7 @@ def apply(self, person_id, squeeze_factor): # rdt is offered as part of the treatment package # Log the test: line-list of summary information about each test - fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id) + fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id=person_id) person_details_for_test = { 'person_id': person_id, 'age': df.at[person_id, 'age_years'], @@ -1311,7 +1311,7 @@ def apply(self, person_id, squeeze_factor): # rdt is offered as part of the treatment package # Log the test: line-list of summary information about each test - fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id) + fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id=person_id) person_details_for_test = { 'person_id': person_id, 'age': df.at[person_id, 'age_years'], diff --git a/src/tlo/methods/measles.py b/src/tlo/methods/measles.py index b6955ff9d7..5d2c6dcc53 100644 --- a/src/tlo/methods/measles.py +++ b/src/tlo/methods/measles.py @@ -442,7 +442,7 @@ def apply(self, person_id, squeeze_factor): data=f"HSI_Measles_Treatment: treat person {person_id} for measles") df = self.sim.population.props - symptoms = self.sim.modules["SymptomManager"].has_what(person_id) + symptoms = self.sim.modules["SymptomManager"].has_what(person_id=person_id) # for non-complicated measles item_codes = [self.module.consumables['vit_A']] diff --git a/src/tlo/methods/oesophagealcancer.py b/src/tlo/methods/oesophagealcancer.py index 1961aa340e..8adc0614e1 100644 --- a/src/tlo/methods/oesophagealcancer.py +++ b/src/tlo/methods/oesophagealcancer.py @@ -681,7 +681,7 @@ def apply(self, person_id, squeeze_factor): return hs.get_blank_appt_footprint() # Check that this event has been called for someone with the symptom dysphagia - assert 'dysphagia' in self.sim.modules['SymptomManager'].has_what(person_id) + assert 'dysphagia' in self.sim.modules['SymptomManager'].has_what(person_id=person_id) # If the person is already diagnosed, then take no action: if not pd.isnull(df.at[person_id, "oc_date_diagnosis"]): diff --git a/src/tlo/methods/other_adult_cancers.py b/src/tlo/methods/other_adult_cancers.py index 5999792393..5aad8f971a 100644 --- a/src/tlo/methods/other_adult_cancers.py +++ b/src/tlo/methods/other_adult_cancers.py @@ -685,7 +685,7 @@ def apply(self, person_id, squeeze_factor): return hs.get_blank_appt_footprint() # Check that this event has been called for someone with the symptom other_adult_ca_symptom - assert 'early_other_adult_ca_symptom' in self.sim.modules['SymptomManager'].has_what(person_id) + assert 'early_other_adult_ca_symptom' in self.sim.modules['SymptomManager'].has_what(person_id=person_id) # If the person is already diagnosed, then take no action: if not pd.isnull(df.at[person_id, "oac_date_diagnosis"]): diff --git a/src/tlo/methods/prostate_cancer.py b/src/tlo/methods/prostate_cancer.py index 8bb7fd82ef..dbbe2c427f 100644 --- a/src/tlo/methods/prostate_cancer.py +++ b/src/tlo/methods/prostate_cancer.py @@ -719,7 +719,7 @@ def apply(self, person_id, squeeze_factor): return hs.get_blank_appt_footprint() # Check that this event has been called for someone with the urinary symptoms - assert 'urinary' in self.sim.modules['SymptomManager'].has_what(person_id) + assert 'urinary' in self.sim.modules['SymptomManager'].has_what(person_id=person_id) # If the person is already diagnosed, then take no action: if not pd.isnull(df.at[person_id, "pc_date_diagnosis"]): @@ -767,7 +767,7 @@ def apply(self, person_id, squeeze_factor): return hs.get_blank_appt_footprint() # Check that this event has been called for someone with the pelvic pain - assert 'pelvic_pain' in self.sim.modules['SymptomManager'].has_what(person_id) + assert 'pelvic_pain' in self.sim.modules['SymptomManager'].has_what(person_id=person_id) # If the person is already diagnosed, then take no action: if not pd.isnull(df.at[person_id, "pc_date_diagnosis"]): diff --git a/src/tlo/methods/symptommanager.py b/src/tlo/methods/symptommanager.py index 26f6aa7ee4..67389e283e 100644 --- a/src/tlo/methods/symptommanager.py +++ b/src/tlo/methods/symptommanager.py @@ -11,9 +11,11 @@ * The probability of spurious symptoms is not informed by data. """ +from __future__ import annotations + from collections import defaultdict from pathlib import Path -from typing import Sequence, Union +from typing import TYPE_CHECKING, List, Optional, Sequence, Union import numpy as np import pandas as pd @@ -23,6 +25,9 @@ from tlo.methods import Metadata from tlo.util import BitsetHandler +if TYPE_CHECKING: + from tlo.population import IndividualProperties + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -460,33 +465,81 @@ def who_not_have(self, symptom_string: str) -> pd.Index: ) ] - def has_what(self, person_id, disease_module: Module = None): + def has_what( + self, + person_id: Optional[int] = None, + individual_details: Optional[IndividualProperties] = None, + disease_module: Optional[Module] = None, + ) -> List[str]: """ This is a helper function that will give a list of strings for the symptoms that a _single_ person is currently experiencing. - Optionally can specify disease_module_name to limit to the symptoms caused by that disease module - :param person_id: the person_of of interest - :param disease_module: (optional) disease module of interest - :return: list of strings for the symptoms that are currently being experienced - """ + If working in a `tlo.population.IndividualProperties` context, one can pass the context object + instead of supplying the person's DataFrame index. + Note that at least one of these inputs must be passed as a keyword argument however. + In the event that both arguments are passed, the individual_details argument takes precedence over the person_id. - assert isinstance(person_id, (int, np.integer)), 'person_id must be a single integer for one particular person' + Optionally can specify disease_module_name to limit to the symptoms caused by that disease module. - df = self.sim.population.props - assert df.at[person_id, 'is_alive'], "The person is not alive" - - if disease_module is not None: - assert disease_module.name in ([self.name] + self.recognised_module_names), \ - "Disease Module Name is not recognised" - sy_columns = [self.get_column_name_for_symptom(s) for s in self.symptom_names] - person_has = self.bsh.has( - [person_id], disease_module.name, first=True, columns=sy_columns - ) - return [s for s in self.symptom_names if person_has[f'sy_{s}']] + :param person_id: the person_of of interest. + :param individual_details: `tlo.population.IndividualProperties` object for the person of interest. + :param disease_module: (optional) disease module of interest. + :return: list of strings for the symptoms that are currently being experienced. + """ + assert ( + disease_module.name in ([self.name] + self.recognised_module_names) + if disease_module is not None + else True + ), "Disease Module Name is not recognised" + + if individual_details is not None: + # We are working in an IndividualDetails context, avoid lookups to the + # population DataFrame as we have this context stored already. + assert individual_details["is_alive"], "The person is not alive" + + if disease_module is not None: + int_repr = self.bsh._element_to_int_map[disease_module.name] + return [ + symptom + for symptom in self.symptom_names + if individual_details[ + self.bsh._get_columns(self.get_column_name_for_symptom(symptom)) + ] + & int_repr + != 0 + ] + else: + return [ + symptom + for symptom in self.symptom_names + if individual_details[self.get_column_name_for_symptom(symptom)] > 0 + ] else: - symptom_cols = df.loc[person_id, [f'sy_{s}' for s in self.symptom_names]] - return symptom_cols.index[symptom_cols > 0].str.removeprefix("sy_").to_list() + assert isinstance( + person_id, (int, np.integer) + ), "person_id must be a single integer for one particular person" + + df = self.sim.population.props + assert df.at[person_id, "is_alive"], "The person is not alive" + + if disease_module is not None: + sy_columns = [ + self.get_column_name_for_symptom(s) for s in self.symptom_names + ] + person_has = self.bsh.has( + [person_id], disease_module.name, first=True, columns=sy_columns + ) + return [s for s in self.symptom_names if person_has[f"sy_{s}"]] + else: + symptom_cols = df.loc[ + person_id, [f"sy_{s}" for s in self.symptom_names] + ] + return ( + symptom_cols.index[symptom_cols > 0] + .str.removeprefix("sy_") + .to_list() + ) def have_what(self, person_ids: Sequence[int]): """Find the set of symptoms for a list of person_ids. diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py index aa62f3ea8a..8b7a061586 100644 --- a/src/tlo/methods/tb.py +++ b/src/tlo/methods/tb.py @@ -1698,7 +1698,7 @@ def apply(self, person_id, squeeze_factor): # check if patient has: cough, fever, night sweat, weight loss # if none of the above conditions are present, no further action - persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id) + persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id=person_id) if not any(x in self.module.symptom_list for x in persons_symptoms): return self.make_appt_footprint({}) @@ -1961,7 +1961,7 @@ def apply(self, person_id, squeeze_factor): # check if patient has: cough, fever, night sweat, weight loss set_of_symptoms_that_indicate_tb = set(self.module.symptom_list) - persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id) + persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id=person_id) if not set_of_symptoms_that_indicate_tb.intersection(persons_symptoms): # if none of the above conditions are present, no further action @@ -2465,7 +2465,7 @@ def apply(self, person_id, squeeze_factor): return # if currently have symptoms of TB, refer for screening/testing - persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id) + persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id=person_id) if any(x in self.module.symptom_list for x in persons_symptoms): self.sim.modules["HealthSystem"].schedule_hsi_event( diff --git a/tests/test_alri.py b/tests/test_alri.py index 0fba5fea8d..fcce8b4b42 100644 --- a/tests/test_alri.py +++ b/tests/test_alri.py @@ -435,7 +435,11 @@ def __will_die_of_alri(**kwargs): assert pd.isnull(person['ri_scheduled_death_date']) # Check that they have some symptoms caused by ALRI - assert 0 < len(sim.modules['SymptomManager'].has_what(person_id, sim.modules['Alri'])) + assert 0 < len( + sim.modules["SymptomManager"].has_what( + person_id=person_id, disease_module=sim.modules["Alri"] + ) + ) # Check that there is a AlriNaturalRecoveryEvent scheduled for this person: recov_event_tuple = [event_tuple for event_tuple in sim.find_events_for_person(person_id) if @@ -458,7 +462,11 @@ def __will_die_of_alri(**kwargs): assert pd.isnull(person['ri_scheduled_death_date']) # check they they have no symptoms: - assert 0 == len(sim.modules['SymptomManager'].has_what(person_id, sim.modules['Alri'])) + assert 0 == len( + sim.modules["SymptomManager"].has_what( + person_id=person_id, disease_module=sim.modules["Alri"] + ) + ) # check it's logged (one infection + one recovery) assert 1 == sim.modules['Alri'].logging_event.trackers['incident_cases'].report_current_total() diff --git a/tests/test_cardiometabolicdisorders.py b/tests/test_cardiometabolicdisorders.py index a40fdad69b..977caa4c91 100644 --- a/tests/test_cardiometabolicdisorders.py +++ b/tests/test_cardiometabolicdisorders.py @@ -770,7 +770,7 @@ def test_hsi_emergency_events(seed): assert pd.isnull(df.at[person_id, f'nc_{event}_scheduled_date_death']) assert isinstance(sim.modules['HealthSystem'].HSI_EVENT_QUEUE[0].hsi_event, HSI_CardioMetabolicDisorders_StartWeightLossAndMedication) - assert f"{event}_damage" not in sim.modules['SymptomManager'].has_what(person_id) + assert f"{event}_damage" not in sim.modules['SymptomManager'].has_what(person_id=person_id) def test_no_availability_of_consumables_for_conditions(seed): diff --git a/tests/test_copd.py b/tests/test_copd.py index 6c8b8a0917..b47d803529 100644 --- a/tests/test_copd.py +++ b/tests/test_copd.py @@ -211,12 +211,12 @@ def test_moderate_exacerbation(): df.at[person_id, 'ch_has_inhaler'] = False # check individuals do not have symptoms before an event is run - assert 'breathless_moderate' not in sim.modules['SymptomManager'].has_what(person_id) + assert 'breathless_moderate' not in sim.modules['SymptomManager'].has_what(person_id=person_id) # run Copd Exacerbation event on an individual and confirm they now have a # non-emergency symptom(breathless moderate) copd.CopdExacerbationEvent(copd_module, person_id, severe=False).run() - assert 'breathless_moderate' in sim.modules['SymptomManager'].has_what(person_id) + assert 'breathless_moderate' in sim.modules['SymptomManager'].has_what(person_id=person_id) # Run health seeking behavior event and check non-emergency care is sought hsp = HealthSeekingBehaviourPoll(sim.modules['HealthSeekingBehaviour']) @@ -259,13 +259,15 @@ def test_severe_exacerbation(): df.at[person_id, 'ch_has_inhaler'] = False # check an individual do not have emergency symptoms before an event is run - assert 'breathless_severe' not in sim.modules['SymptomManager'].has_what(person_id) + assert 'breathless_severe' not in sim.modules['SymptomManager'].has_what(person_id=person_id) # schedule exacerbations event setting severe to True. This will ensure the individual has severe exacerbation copd.CopdExacerbationEvent(copd_module, person_id, severe=True).run() # severe exacerbation should lead to severe symptom(breathless severe in this case). check this is true - assert 'breathless_severe' in sim.modules['SymptomManager'].has_what(person_id, copd_module) + assert "breathless_severe" in sim.modules["SymptomManager"].has_what( + person_id=person_id, disease_module=copd_module + ) # # Run health seeking behavior event and check emergency care is sought hsp = HealthSeekingBehaviourPoll(module=sim.modules['HealthSeekingBehaviour']) @@ -420,13 +422,15 @@ def test_referral_logic(): df.at[person_id, 'ch_has_inhaler'] = False # check an individual do not have emergency symptoms before an event is run - assert 'breathless_severe' not in sim.modules['SymptomManager'].has_what(person_id) + assert 'breathless_severe' not in sim.modules['SymptomManager'].has_what(person_id=person_id) # schedule exacerbations event setting severe to True. This will ensure the individual has severe exacerbation copd.CopdExacerbationEvent(copd_module, person_id, severe=True).run() # severe exacerbation should lead to severe symptom(breathless severe in this case). check this is true - assert 'breathless_severe' in sim.modules['SymptomManager'].has_what(person_id, copd_module) + assert "breathless_severe" in sim.modules["SymptomManager"].has_what( + person_id=person_id, disease_module=copd_module + ) # Run health seeking behavior event and check emergency care is sought hsp = HealthSeekingBehaviourPoll(module=sim.modules['HealthSeekingBehaviour']) diff --git a/tests/test_hiv.py b/tests/test_hiv.py index 47ef0d2083..5a27cf2c33 100644 --- a/tests/test_hiv.py +++ b/tests/test_hiv.py @@ -224,7 +224,7 @@ def test_generation_of_natural_history_process_no_art(seed): # run the AIDS onset event for this person: aids_event.apply(person_id) - assert "aids_symptoms" in sim.modules['SymptomManager'].has_what(person_id) + assert "aids_symptoms" in sim.modules['SymptomManager'].has_what(person_id=person_id) # find the AIDS death event for this person date_aids_death_event, aids_death_event = \ @@ -274,7 +274,7 @@ def test_generation_of_natural_history_process_with_art_before_aids(seed): assert [] == [ev for ev in sim.find_events_for_person(person_id) if isinstance(ev[1], hiv.HivAidsDeathEvent)] # check no AIDS symptoms for this person - assert "aids_symptoms" not in sim.modules['SymptomManager'].has_what(person_id) + assert "aids_symptoms" not in sim.modules['SymptomManager'].has_what(person_id=person_id) def test_generation_of_natural_history_process_with_art_after_aids(seed): @@ -312,7 +312,7 @@ def test_generation_of_natural_history_process_with_art_after_aids(seed): date_aids_death_event, aids_death_event = \ [ev for ev in sim.find_events_for_person(person_id) if isinstance(ev[1], hiv.HivAidsDeathEvent)][0] assert date_aids_death_event > sim.date - assert "aids_symptoms" in sim.modules['SymptomManager'].has_what(person_id) + assert "aids_symptoms" in sim.modules['SymptomManager'].has_what(person_id=person_id) # Put the person on ART with VL suppression prior to the AIDS death (but following AIDS onset) df.at[person_id, 'hv_art'] = "on_VL_suppressed" @@ -516,7 +516,7 @@ def test_aids_symptoms_lead_to_treatment_being_initiated(seed): aids_event.apply(person_id) # Confirm that they have aids symptoms and an AIDS death schedule - assert 'aids_symptoms' in sim.modules['SymptomManager'].has_what(person_id) + assert 'aids_symptoms' in sim.modules['SymptomManager'].has_what(person_id=person_id) assert 1 == len( [ev[0] for ev in sim.find_events_for_person(person_id) if isinstance(ev[1], hiv.HivAidsTbDeathEvent)]) diff --git a/tests/test_malaria.py b/tests/test_malaria.py index 6fb185c433..2b16da0000 100644 --- a/tests/test_malaria.py +++ b/tests/test_malaria.py @@ -268,7 +268,7 @@ def test_dx_algorithm_for_malaria_outcomes_clinical( add_or_remove='+' ) - assert "fever" in sim.modules["SymptomManager"].has_what(person_id) + assert "fever" in sim.modules["SymptomManager"].has_what(person_id=person_id) def diagnosis_function(tests, use_dict: bool = False, report_tried: bool = False): return hsi_event.healthcare_system.dx_manager.run_dx_test( @@ -346,7 +346,7 @@ def make_blank_simulation(): add_or_remove='+' ) - assert "fever" in sim.modules["SymptomManager"].has_what(person_id) + assert "fever" in sim.modules["SymptomManager"].has_what(person_id=person_id) def diagnosis_function(tests, use_dict: bool = False, report_tried: bool = False): return hsi_event.healthcare_system.dx_manager.run_dx_test( @@ -517,7 +517,7 @@ def test_individual_testing_and_treatment(sim): pollevent.run() assert not pd.isnull(df.at[person_id, "ma_date_symptoms"]) - assert set(sim.modules['SymptomManager'].has_what(person_id)) == {"fever", "headache", "vomiting", "stomachache"} + assert set(sim.modules['SymptomManager'].has_what(person_id=person_id)) == {"fever", "headache", "vomiting", "stomachache"} # check rdt is scheduled date_event, event = [ @@ -560,7 +560,7 @@ def test_individual_testing_and_treatment(sim): pollevent = malaria.MalariaUpdateEvent(module=sim.modules['Malaria']) pollevent.apply(sim.population) - assert sim.modules['SymptomManager'].has_what(person_id) == [] + assert sim.modules['SymptomManager'].has_what(person_id=person_id) == [] # check no rdt is scheduled assert "malaria.HSI_Malaria_rdt" not in sim.modules['HealthSystem'].find_events_for_person(person_id) diff --git a/tests/test_symptommanager.py b/tests/test_symptommanager.py index 85c7156902..73ea7619d0 100644 --- a/tests/test_symptommanager.py +++ b/tests/test_symptommanager.py @@ -1,5 +1,8 @@ +from __future__ import annotations + import os from pathlib import Path +from typing import TYPE_CHECKING, List import pytest from pandas import DateOffset @@ -24,6 +27,9 @@ SymptomManager_SpuriousSymptomOnset, ) +if TYPE_CHECKING: + from tlo.methods.symptommanager import SymptomManager + try: resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' except NameError: @@ -187,8 +193,9 @@ def test_adding_quering_and_removing_symptoms(seed): assert set(has_symp) == set(ids) for person_id in ids: - assert symp in sim.modules['SymptomManager'].has_what(person_id=person_id, - disease_module=sim.modules['Mockitis']) + assert symp in sim.modules["SymptomManager"].has_what( + person_id=person_id, disease_module=sim.modules["Mockitis"] + ) # Check cause of the symptom: for person in ids: @@ -203,6 +210,103 @@ def test_adding_quering_and_removing_symptoms(seed): assert list() == sim.modules['SymptomManager'].who_has(symp) +@pytest.mark.parametrize( + "supply_disease_module", + [ + pytest.param(False, id="disease_module kwarg NOT supplied"), + pytest.param(True, id="disease_module kwarg supplied"), + ], +) +def test_has_what_via_individual_properties(seed, supply_disease_module: bool): + """ + Test that the has_what method returns the same symptoms for an individual + when supplied a person_id and the individual_properties context for that + same person. + + Test the case when the optional disease_module kwarg is supplied as well. + + We will create 3 'dummy' symptoms and select 8 individuals in the + population to infect with these symptoms; in the following combinations: + + id has_symp1 has_symp2 has_symp3 + 0 1 1 1 + 1 1 1 0 + 2 1 0 1 + 3 1 0 0 + 4 0 1 1 + 5 0 1 0 + 6 0 0 1 + 7 0 0 0 + + We will then assert that has_what returns the expected symptoms for the + individuals, and that supplying either the person_id keyword or the + individual_properties keyword gives the same answer. + """ + sim = Simulation(start_date=start_date, seed=seed) + sim.register( + demography.Demography(resourcefilepath=resourcefilepath), + enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath), + healthsystem.HealthSystem(resourcefilepath=resourcefilepath, disable=True), + symptommanager.SymptomManager(resourcefilepath=resourcefilepath), + healthseekingbehaviour.HealthSeekingBehaviour( + resourcefilepath=resourcefilepath + ), + simplified_births.SimplifiedBirths(resourcefilepath=resourcefilepath), + mockitis.Mockitis(), + chronicsyndrome.ChronicSyndrome(), + ) + disease_module: mockitis.Mockitis = sim.modules["Mockitis"] + symptom_manager: SymptomManager = sim.modules["SymptomManager"] + + # Generate the symptoms and select the people to infect + n_symptoms = 3 + n_patients = 2 ** n_symptoms + symptoms = [f"test_symptom{i}" for i in range(n_symptoms)] + symptom_manager.register_symptom(*[Symptom(name=symptom) for symptom in symptoms]) + + # Create the initial population after generating extra symptoms, so that they are registered + sim.make_initial_population(n=popsize) + df = sim.population.props + + # Infect the people with the corresponding symptoms + persons_infected_with: List[int] = [ + id for id in sim.rng.choice(list(df.index[df.is_alive]), n_patients) + ] + for i, id in enumerate(persons_infected_with): + bin_rep = format(i, f"0{n_symptoms}b") + for symptom_number, digit in enumerate(bin_rep): + if digit == "1": + symptom_manager.change_symptom( + symptom_string=symptoms[symptom_number], + person_id=[id], + add_or_remove="+", + disease_module=disease_module, + ) + + # Now check that has_what returns the same (correct!) arguments when supplied with + # individual_properties and person_id. + for person_id in persons_infected_with: + symptoms_via_pid = symptom_manager.has_what( + person_id=person_id, + disease_module=disease_module if supply_disease_module else None, + ) + with sim.population.individual_properties( + person_id, read_only=True + ) as individual_properties: + symptoms_via_iprops = symptom_manager.has_what( + individual_details=individual_properties, + disease_module=disease_module if supply_disease_module else None, + ) + + # Assert all returned symptoms are in agreement + assert len(symptoms_via_pid) == len( + symptoms_via_iprops + ), "Method does not return same number of symptoms." + assert set(symptoms_via_pid) == set( + symptoms_via_iprops + ), "Method does not return the same symptoms" + + def test_baby_born_has_no_symptoms(seed): sim = Simulation(start_date=start_date, seed=seed) @@ -227,7 +331,7 @@ def test_baby_born_has_no_symptoms(seed): person_id = sim.do_birth(mother_id) # check that the new person does not have symptoms: - assert [] == sim.modules['SymptomManager'].has_what(person_id) + assert [] == sim.modules['SymptomManager'].has_what(person_id=person_id) def test_auto_onset_symptom(seed): @@ -250,7 +354,7 @@ def test_auto_onset_symptom(seed): sim.population.props.loc[person_id, 'is_alive'] = True for symptom in sm.symptom_names: sim.population.props.loc[person_id, sm.get_column_name_for_symptom(symptom)] = 0 - assert 0 == len(sm.has_what(person_id)) + assert 0 == len(sm.has_what(person_id=person_id)) def get_events_in_sim(): return [ev for ev in sim.event_queue.queue if (person_id in ev[3].person_id)] @@ -273,7 +377,7 @@ def get_events_in_sim(): ) # check that the symptom is not imposed - assert 0 == len(sm.has_what(person_id)) + assert 0 == len(sm.has_what(person_id=person_id)) # get the future events for this person (should be just the auto-onset event) assert 1 == len(get_events_in_sim()) @@ -285,7 +389,7 @@ def get_events_in_sim(): # run the events and check for the changing of symptoms sim.date = date_of_onset onset[3].apply(sim.population) - assert symptom_string in sm.has_what(person_id) + assert symptom_string in sm.has_what(person_id=person_id) # get the future events for this person (should now include the auto-resolve event) assert 2 == len(get_events_in_sim()) @@ -295,7 +399,7 @@ def get_events_in_sim(): assert isinstance(resolve[3], SymptomManager_AutoResolveEvent) resolve[3].apply(sim.population) - assert 0 == len(sm.has_what(person_id)) + assert 0 == len(sm.has_what(person_id=person_id)) def test_nonemergency_spurious_symptoms_during_simulation(seed): @@ -504,13 +608,26 @@ def test_has_what( df.is_alive & (df[symptom_manager.get_column_name_for_symptom(symptom)] > 0) ][0] - assert symptom in symptom_manager.has_what(person_with_symptom) + assert symptom in symptom_manager.has_what(person_id=person_with_symptom) person_without_symptom = df.index[ df.is_alive & (df[symptom_manager.get_column_name_for_symptom(symptom)] == 0) ][0] - assert symptom not in symptom_manager.has_what(person_without_symptom) - + assert symptom not in symptom_manager.has_what(person_id=person_without_symptom) + + # Do the same checks but using an IndividualDetails context + with simulation.population.individual_properties( + person_with_symptom, read_only=True + ) as with_symptom_properties: + assert symptom in symptom_manager.has_what( + individual_details=with_symptom_properties + ) + with simulation.population.individual_properties( + person_without_symptom, read_only=True + ) as without_symptom_properties: + assert symptom not in symptom_manager.has_what( + individual_details=without_symptom_properties + ) def test_has_what_disease_module( symptom_manager, disease_module, disease_module_symptoms, simulation @@ -522,12 +639,16 @@ def test_has_what_disease_module( df.is_alive & (df[symptom_manager.get_column_name_for_symptom(symptom)] > 0) ][0] - assert symptom in symptom_manager.has_what(person_with_symptom, disease_module) + assert symptom in symptom_manager.has_what( + person_id=person_with_symptom, disease_module=disease_module + ) person_without_symptom = df.index[ df.is_alive & (df[symptom_manager.get_column_name_for_symptom(symptom)] == 0) ][0] - assert symptom not in symptom_manager.has_what(person_without_symptom, disease_module) + assert symptom not in symptom_manager.has_what( + person_id=person_without_symptom, disease_module=disease_module + ) def test_have_what( diff --git a/tests/test_tb.py b/tests/test_tb.py index 0434c70069..66d5abd60e 100644 --- a/tests/test_tb.py +++ b/tests/test_tb.py @@ -576,7 +576,7 @@ def test_children_referrals(seed): duration_in_days=None, ) - assert set(sim.modules['SymptomManager'].has_what(person_id)) == symptom_list + assert set(sim.modules['SymptomManager'].has_what(person_id=person_id)) == symptom_list # run HSI_Tb_ScreeningAndRefer and check outcomes sim.modules['HealthSystem'].schedule_hsi_event( @@ -1036,7 +1036,7 @@ def test_hsi_scheduling(seed): duration_in_days=None, ) - assert set(sim.modules['SymptomManager'].has_what(person_id)) == symptom_list + assert set(sim.modules['SymptomManager'].has_what(person_id=person_id)) == symptom_list hsi_event = tb.HSI_Tb_ScreeningAndRefer(person_id=person_id, module=sim.modules['Tb']) hsi_event.run(squeeze_factor=0) @@ -1080,7 +1080,7 @@ def test_hsi_scheduling(seed): duration_in_days=None, ) - assert set(sim.modules['SymptomManager'].has_what(person_id)) == symptom_list + assert set(sim.modules['SymptomManager'].has_what(person_id=person_id)) == symptom_list hsi_event = tb.HSI_Tb_ScreeningAndRefer(person_id=person_id, module=sim.modules['Tb']) hsi_event.run(squeeze_factor=0) @@ -1125,7 +1125,7 @@ def test_hsi_scheduling(seed): duration_in_days=None, ) - assert set(sim.modules['SymptomManager'].has_what(person_id)) == symptom_list + assert set(sim.modules['SymptomManager'].has_what(person_id=person_id)) == symptom_list hsi_event = tb.HSI_Tb_ScreeningAndRefer(person_id=person_id, module=sim.modules['Tb']) hsi_event.run(squeeze_factor=0) From 625b4d91edd7b77c382b9860e22aa86d78d53ce5 Mon Sep 17 00:00:00 2001 From: Matt Graham Date: Wed, 24 Jul 2024 09:35:52 +0100 Subject: [PATCH 11/19] Ensure log entries use consistent ordering and types for columns (#1404) * Sort structured log dataframe entries and raise error on inconsistent columns * Sort diff entries in logging error message * Add helper for converting dataframe row to dict for logging * Add extra JSON encoding rules for logging * Fix errors in modules due to unstable log columns * More unstable log columns fixes * Fix further instance of misaligned log entry key * Fix measles incidence age range log entry float / int instability * Ensure HSI event priorities are ints * Handle all pandas extension types in logging encoder + helper function * Use helper function to ensure type stability in equipment logging * Fix isort spacing issues * More manual fixes for log entry float/int type instability * Ensure groupby on age_years includes combinations with zero counts * Ensure stunting log contains all age_years combinations * Normalize NumPy scalar types and refactor logging * Further logging refactoring * Updating logging tests * Ensure numeric dict keys use natural sort order * Automatically convert NumPy strings to Python type * Fix bug in length 1 extension array type handling * Add helper function for logging group by counts * Remove new line to satisfy isort * Use helper function to log person dict * Make district_num_of_residence property categorical * Make dummy missing facility level value str * Fix int/float switching in TB incidence logging * Fix import order * Have pylint ignore dynamic property access * Ensure type stability in CMD logging * Fix end to end logging tests * Make inconsistent logging columns error a warning * Remove superfluous logger level check * Fix merge conflict placeholders left in bad merge --- src/tlo/logging/__init__.py | 32 +- src/tlo/logging/core.py | 454 +++++++---- src/tlo/logging/encoding.py | 11 +- src/tlo/logging/helpers.py | 113 ++- src/tlo/methods/cardio_metabolic_disorders.py | 4 +- .../methods/care_of_women_during_pregnancy.py | 4 +- src/tlo/methods/consumables.py | 25 +- src/tlo/methods/demography.py | 19 +- src/tlo/methods/depression.py | 4 +- src/tlo/methods/enhanced_lifestyle.py | 48 +- src/tlo/methods/epilepsy.py | 6 +- src/tlo/methods/equipment.py | 6 +- src/tlo/methods/healthsystem.py | 11 +- src/tlo/methods/hiv.py | 6 +- src/tlo/methods/labour.py | 15 +- src/tlo/methods/malaria.py | 99 ++- src/tlo/methods/measles.py | 4 +- src/tlo/methods/newborn_outcomes.py | 2 +- src/tlo/methods/rti.py | 54 +- src/tlo/methods/stunting.py | 4 +- src/tlo/methods/tb.py | 18 +- src/tlo/simulation.py | 11 +- tests/test_logging.py | 742 ++++++++++++++---- tests/test_logging_end_to_end.py | 38 +- 24 files changed, 1196 insertions(+), 534 deletions(-) diff --git a/src/tlo/logging/__init__.py b/src/tlo/logging/__init__.py index e17e5c37b5..7f1447f037 100644 --- a/src/tlo/logging/__init__.py +++ b/src/tlo/logging/__init__.py @@ -1,7 +1,27 @@ -from .core import CRITICAL, DEBUG, FATAL, INFO, WARNING, disable, getLogger -from .helpers import init_logging, set_logging_levels, set_output_file, set_simulation +from .core import ( + CRITICAL, + DEBUG, + FATAL, + INFO, + WARNING, + disable, + getLogger, + initialise, + reset, + set_output_file, +) +from .helpers import set_logging_levels -__all__ = ['CRITICAL', 'DEBUG', 'FATAL', 'INFO', 'WARNING', 'disable', 'getLogger', - 'set_output_file', 'init_logging', 'set_simulation', 'set_logging_levels'] - -init_logging() +__all__ = [ + "CRITICAL", + "DEBUG", + "FATAL", + "INFO", + "WARNING", + "disable", + "getLogger", + "initialise", + "reset", + "set_output_file", + "set_logging_levels", +] diff --git a/src/tlo/logging/core.py b/src/tlo/logging/core.py index e870e1f179..dc3beaf2f1 100644 --- a/src/tlo/logging/core.py +++ b/src/tlo/logging/core.py @@ -1,217 +1,361 @@ +from __future__ import annotations + import hashlib import json import logging as _logging -from typing import Union +import sys +import warnings +from functools import partialmethod +from pathlib import Path +from typing import Any, Callable, List, Optional, TypeAlias, Union +import numpy as np import pandas as pd from tlo.logging import encoding +LogLevel: TypeAlias = int +LogData: TypeAlias = Union[str, dict, list, set, tuple, pd.DataFrame, pd.Series] +SimulationDateGetter: TypeAlias = Callable[[], str] + +CRITICAL = _logging.CRITICAL +DEBUG = _logging.DEBUG +FATAL = _logging.FATAL +INFO = _logging.INFO +WARNING = _logging.WARNING -def disable(level): +_DEFAULT_LEVEL = INFO + +_DEFAULT_FORMATTER = _logging.Formatter("%(message)s") + + +class InconsistentLoggedColumnsWarning(UserWarning): + """Warning raised when structured log entry has different columns from header.""" + + +def _mock_simulation_date_getter() -> str: + return "0000-00-00T00:00:00" + + +_get_simulation_date: SimulationDateGetter = _mock_simulation_date_getter +_loggers: dict[str, Logger] = {} + + +def initialise( + add_stdout_handler: bool = True, + simulation_date_getter: SimulationDateGetter = _mock_simulation_date_getter, + root_level: LogLevel = WARNING, + stdout_handler_level: LogLevel = DEBUG, + formatter: _logging.Formatter = _DEFAULT_FORMATTER, +) -> None: + """Initialise logging system and set up root `tlo` logger. + + :param add_stdout_handler: Whether to add a handler to output log entries to stdout. + :param simulation_date_getter: Zero-argument function returning simulation date as + string in ISO format to use in log entries. Defaults to function returning a + a fixed dummy date for use before a simulation has been initialised. + :param root_level: Logging level for root `tlo` logger. + :param formatter: Formatter to use for logging to stdout. + """ + global _get_simulation_date, _loggers + _get_simulation_date = simulation_date_getter + for logger in _loggers.values(): + logger.reset_attributes() + root_logger = getLogger("tlo") + root_logger.setLevel(root_level) + if add_stdout_handler: + handler = _logging.StreamHandler(sys.stdout) + handler.setLevel(stdout_handler_level) + handler.setFormatter(formatter) + root_logger.handlers = [ + h + for h in root_logger.handlers + if not (isinstance(h, _logging.StreamHandler) and h.stream is sys.stdout) + ] + root_logger.addHandler(handler) + + +def reset(): + """Reset global logging state to values at initial import.""" + global _get_simulation_date, _loggers + while len(_loggers) > 0: + name, _ = _loggers.popitem() + _logging.root.manager.loggerDict.pop(name, None) # pylint: disable=E1101 + _loggers.clear() + _get_simulation_date = _mock_simulation_date_getter + + +def set_output_file( + log_path: Path, + formatter: _logging.Formatter = _DEFAULT_FORMATTER, +) -> _logging.FileHandler: + """Add file handler to logger. + + :param log_path: Path for file. + :return: File handler object. + """ + file_handler = _logging.FileHandler(log_path) + file_handler.setFormatter(formatter) + logger = getLogger("tlo") + logger.handlers = [ + h for h in logger.handlers if not isinstance(h, _logging.FileHandler) + ] + logger.addHandler(file_handler) + return file_handler + + +def disable(level: LogLevel) -> None: + """Disable all logging calls of specified level and below.""" _logging.disable(level) -def getLogger(name='tlo'): +def getLogger(name: str = "tlo") -> Logger: """Returns a TLO logger of the specified name""" - if name not in _LOGGERS: - _LOGGERS[name] = Logger(name) - return _LOGGERS[name] + if name not in _loggers: + _loggers[name] = Logger(name) + return _loggers[name] + + +def _numeric_or_str_sort_key(value): + """Key function to sort mixture of numeric and string items. + + Orders non-string values first and then string values, assuming ascending order. + """ + return isinstance(value, str), value + + +def _convert_keys_to_strings_and_sort(data: dict) -> dict[str, Any]: + """Convert all dictionary keys to strings and sort dictionary by key.""" + # Sort by mix of numeric or string keys _then_ convert all keys to strings to + # ensure stringified numeric keys have natural numeric ordering, for example + # '1', '2', '10' not '1', '10', '2' + sorted_data = dict( + (str(k), v) + for k, v in sorted(data.items(), key=lambda i: _numeric_or_str_sort_key(i[0])) + ) + if len(sorted_data) != len(data): + raise ValueError( + f"At least one pair of keys in data dictionary {data} map to same string." + ) + return sorted_data + + +def _sort_set_with_numeric_or_str_elements(data: set) -> list: + """Sort a set with elements that may be either strings or numeric types.""" + return sorted(data, key=_numeric_or_str_sort_key) + + +def _get_log_data_as_dict(data: LogData) -> dict: + """Convert log data to a dictionary if it isn't already""" + if isinstance(data, dict): + return _convert_keys_to_strings_and_sort(data) + if isinstance(data, pd.DataFrame): + if len(data) == 1: + data_dict = data.iloc[0].to_dict() + return _convert_keys_to_strings_and_sort(data_dict) + else: + raise ValueError( + "Logging multirow dataframes is not currently supported - " + "if you need this feature let us know" + ) + if isinstance(data, (list, set, tuple, pd.Series)): + if isinstance(data, set): + data = _sort_set_with_numeric_or_str_elements(data) + return {f"item_{index + 1}": value for index, value in enumerate(data)} + if isinstance(data, str): + return {"message": data} + raise ValueError(f"Unexpected type given as data:\n{data}") + + +def _convert_numpy_scalars_to_python_types(data: dict) -> dict: + """Convert NumPy scalar types to suitable standard Python types.""" + return { + key: ( + value.item() if isinstance(value, (np.number, np.bool_, np.str_)) else value + ) + for key, value in data.items() + } + + +def _get_columns_from_data_dict(data: dict) -> dict: + """Get columns dictionary specifying types of data dictionary values.""" + # using type().__name__ so both pandas and stdlib types can be used + return {k: type(v).__name__ for k, v, in data.items()} -class _MockSim: - # used as place holder for any logging that happens before simulation is setup! - class MockDate: - @staticmethod - def isoformat(): - return "0000-00-00T00:00:00" - date = MockDate() +class Logger: + """Logger for structured log messages output by simulation. + Outputs structured log messages in JSON format along with simulation date log entry + was generated at. Log messages are associated with a string key and for each key + the log message data is expected to have a fixed structure: -class Logger: - """A Logger for TLO log messages, with simplified usage. Outputs structured log messages in JSON - format and is connected to the Simulation instance.""" - HASH_LEN = 10 + - Collection like data (tuples, lists, sets) should be of fixed length. + - Mapping like data (dictionaries, pandas series and dataframes) should have a fixed + set of keys and the values should be of fixed data types. - def __init__(self, name: str, level=_logging.NOTSET): + The first log message for a given key will generate a 'header' log entry which + records the structure of the message with subsequent log messages only logging the + values for efficiency, hence the requirement for the structure to remain fixed. + """ - assert name.startswith('tlo'), f'Only logging of tlo modules is allowed; name is {name}' + HASH_LEN = 10 + def __init__(self, name: str, level: LogLevel = _DEFAULT_LEVEL) -> None: + assert name.startswith( + "tlo" + ), f"Only logging of tlo modules is allowed; name is {name}" # we build our logger on top of the standard python logging self._std_logger = _logging.getLogger(name=name) self._std_logger.setLevel(level) - self.name = self._std_logger.name - - # don't propograte messages up from "tlo" to root logger - if name == 'tlo': + # don't propagate messages up from "tlo" to root logger + if name == "tlo": self._std_logger.propagate = False + # the unique identifiers of the structured logging calls for this logger + self._uuids = dict() + # the columns for the structured logging calls for this logger + self._columns = dict() - # the key of the structured logging calls for this logger - self.keys = dict() - - # populated by init_logging(simulation) for the top-level "tlo" logger - self.simulation = _MockSim() - - # a logger should only be using old-style or new-style logging, not a mixture - self.logged_stdlib = False - self.logged_structured = False + def __repr__(self) -> str: + return f"" - # disable logging multirow dataframes until we're confident it's robust - self._disable_dataframe_logging = True - - def __repr__(self): - return f'' + @property + def name(self) -> str: + return self._std_logger.name @property - def handlers(self): + def handlers(self) -> List[_logging.Handler]: return self._std_logger.handlers @property - def level(self): + def level(self) -> LogLevel: return self._std_logger.level @handlers.setter - def handlers(self, handlers): + def handlers(self, handlers: List[_logging.Handler]): self._std_logger.handlers.clear() for handler in handlers: self._std_logger.handlers.append(handler) - def addHandler(self, hdlr): + def addHandler(self, hdlr: _logging.Handler): self._std_logger.addHandler(hdlr=hdlr) - def isEnabledFor(self, level): + def isEnabledFor(self, level: LogLevel) -> bool: return self._std_logger.isEnabledFor(level) - def reset_attributes(self): + def reset_attributes(self) -> None: """Reset logger attributes to an unset state""" # clear all logger settings self.handlers.clear() - self.keys.clear() - self.simulation = _MockSim() - # boolean attributes used for now, can be removed after transition to structured logging - self.logged_stdlib = False - self.logged_structured = False - self.setLevel(INFO) - - def setLevel(self, level): + self._uuids.clear() + self._columns.clear() + self.setLevel(_DEFAULT_LEVEL) + + def setLevel(self, level: LogLevel) -> None: self._std_logger.setLevel(level) - def _get_data_as_dict(self, data): - """Convert log data to a dictionary if it isn't already""" - if isinstance(data, dict): - return data - if isinstance(data, pd.DataFrame): - if len(data.index) == 1: - return data.to_dict('records')[0] - elif self._disable_dataframe_logging: - raise ValueError("Logging multirow dataframes is disabled - if you need this feature let us know") - else: - return {'dataframe': data.to_dict('index')} - if isinstance(data, (list, set, tuple, pd.Series)): - return {f'item_{index + 1}': value for index, value in enumerate(data)} - if isinstance(data, str): - return {'message': data} - - raise ValueError(f'Unexpected type given as data:\n{data}') - - def _get_json(self, level, key, data: Union[dict, pd.DataFrame, list, set, tuple, str] = None, description=None): - """Writes structured log message if handler allows this and logging level is allowed - - Will write a header line the first time a new logging key is encountered - Then will only write data rows in later rows for this logging key - - :param level: Level the message is being logged as - :param key: logging key - :param data: data to be logged - :param description: description of this log type - """ - # message level less than than the logger level, early exit - if level < self._std_logger.level: - return + def _get_uuid(self, key: str) -> str: + hexdigest = hashlib.md5(f"{self.name}+{key}".encode()).hexdigest() + return hexdigest[: Logger.HASH_LEN] - data = self._get_data_as_dict(data) - header_json = "" + def _get_json( + self, + level: int, + key: str, + data: Optional[LogData] = None, + description: Optional[str] = None, + ) -> str: + """Writes structured log message if handler allows this and level is allowed. - if key not in self.keys: - # new log key, so create header json row - uuid = hashlib.md5(f"{self.name}+{key}".encode()).hexdigest()[:Logger.HASH_LEN] - self.keys[key] = uuid + Will write a header line the first time a new logging key is encountered. + Then will only write data rows in later rows for this logging key. + + :param level: Level the message is being logged as. + :param key: Logging key. + :param data: Data to be logged. + :param description: Description of this log type. + + :returns: String with JSON-encoded data row and optionally header row. + """ + data = _get_log_data_as_dict(data) + data = _convert_numpy_scalars_to_python_types(data) + header_json = None + if key not in self._uuids: + # new log key, so create header json row + uuid = self._get_uuid(key) + columns = _get_columns_from_data_dict(data) + self._uuids[key] = uuid + self._columns[key] = columns header = { "uuid": uuid, "type": "header", "module": self.name, "key": key, "level": _logging.getLevelName(level), - # using type().__name__ so both pandas and stdlib types can be used - "columns": {key: type(value).__name__ for key, value in data.items()}, - "description": description + "columns": columns, + "description": description, } - header_json = json.dumps(header) + "\n" - - uuid = self.keys[key] - - # create data json row; in DEBUG mode we echo the module and key for easier eyeballing - if self._std_logger.level == DEBUG: - row = {"date": getLogger('tlo').simulation.date.isoformat(), - "module": self.name, - "key": key, - "uuid": uuid, - "values": list(data.values())} + header_json = json.dumps(header) else: - row = {"uuid": uuid, - "date": getLogger('tlo').simulation.date.isoformat(), - "values": list(data.values())} + uuid = self._uuids[key] + columns = _get_columns_from_data_dict(data) + if columns != self._columns[key]: + header_columns = set(self._columns[key].items()) + logged_columns = set(columns.items()) + msg = ( + f"Inconsistent columns in logged values for {self.name} logger " + f"with key {key} compared to header generated from initial log " + f"entry:\n" + f" Columns in header not in logged values are\n" + f" {dict(sorted(header_columns - logged_columns))}\n" + f" Columns in logged values not in header are\n" + f" {dict(sorted(logged_columns - header_columns))}" + ) + warnings.warn( + msg, + InconsistentLoggedColumnsWarning, + # Set stack level so that user is given location of top-level + # {info,warning,debug,critical} convenience method call + stacklevel=3, + ) + + # create data json row + row = { + "uuid": uuid, + "date": _get_simulation_date(), + "values": list(data.values()), + } + if self._std_logger.level == DEBUG: + # in DEBUG mode we echo the module and key for easier eyeballing + row["module"] = self.name + row["key"] = key row_json = json.dumps(row, cls=encoding.PandasEncoder) - return f"{header_json}{row_json}" - - def _make_old_style_msg(self, level, msg): - return f'{level}|{self.name}|{msg}' - - def _check_logging_style(self, is_structured: bool): - """Set booleans for logging type and throw exception if both types of logging haven't been used""" - if is_structured: - self.logged_structured = True - else: - self.logged_stdlib = True - - if self.logged_structured and self.logged_stdlib: - raise ValueError(f"Both oldstyle and structured logging has been used for {self.name}, " - "please update all logging to use structured logging") - - def _check_and_filter(self, msg=None, *args, key=None, data=None, description=None, level, **kwargs): + return row_json if header_json is None else f"{header_json}\n{row_json}" + + def log( + self, + level: LogLevel, + key: str, + data: LogData, + description: Optional[str] = None, + ) -> None: + """Log structured data for a key at specified level with optional description. + + :param level: Level the message is being logged as. + :param key: Logging key. + :param data: Data to be logged. + :param description: Description of this log type. + """ if self._std_logger.isEnabledFor(level): - level_str = _logging.getLevelName(level) # e.g. 'CRITICAL', 'INFO' etc. - level_function = getattr(self._std_logger, level_str.lower()) # e.g. `critical` or `info` methods - if key is None or data is None: - raise ValueError("Structured logging requires `key` and `data` keyword arguments") - self._check_logging_style(is_structured=True) - level_function(self._get_json(level=level, key=key, data=data, description=description)) - - def critical(self, msg=None, *args, key: str = None, - data: Union[dict, pd.DataFrame, list, set, tuple, str] = None, description=None, **kwargs): - self._check_and_filter(msg, *args, key=key, data=data, description=description, level=CRITICAL, **kwargs) - - def debug(self, msg=None, *args, key: str = None, - data: Union[dict, pd.DataFrame, list, set, tuple, str] = None, description=None, **kwargs): - self._check_and_filter(msg, *args, key=key, data=data, description=description, level=DEBUG, **kwargs) - - def info(self, msg=None, *args, key: str = None, - data: Union[dict, pd.DataFrame, list, set, tuple, str] = None, description=None, **kwargs): - self._check_and_filter(msg, *args, key=key, data=data, description=description, level=INFO, **kwargs) - - def warning(self, msg=None, *args, key: str = None, - data: Union[dict, pd.DataFrame, list, set, tuple, str] = None, description=None, **kwargs): - self._check_and_filter(msg, *args, key=key, data=data, description=description, level=WARNING, **kwargs) - - -CRITICAL = _logging.CRITICAL -DEBUG = _logging.DEBUG -FATAL = _logging.FATAL -INFO = _logging.INFO -WARNING = _logging.WARNING - -_FORMATTER = _logging.Formatter('%(message)s') -_LOGGERS = {'tlo': Logger('tlo', WARNING)} + msg = self._get_json( + level=level, key=key, data=data, description=description + ) + self._std_logger.log(level=level, msg=msg) + + critical = partialmethod(log, CRITICAL) + debug = partialmethod(log, DEBUG) + info = partialmethod(log, INFO) + warning = partialmethod(log, WARNING) diff --git a/src/tlo/logging/encoding.py b/src/tlo/logging/encoding.py index 9968ce9cb8..c5db27caa5 100644 --- a/src/tlo/logging/encoding.py +++ b/src/tlo/logging/encoding.py @@ -2,6 +2,7 @@ import numpy as np import pandas as pd +from pandas.api.types import is_extension_array_dtype class PandasEncoder(json.JSONEncoder): @@ -10,16 +11,16 @@ def default(self, obj): # using base classes for numpy numeric types if isinstance(obj, np.floating): return float(obj) - elif isinstance(obj, np.signedinteger): + elif isinstance(obj, np.integer): return int(obj) elif isinstance(obj, pd.Timestamp): return obj.isoformat() - elif isinstance(obj, pd.Categorical): - # assume only only one categorical value per cell - return obj.tolist()[0] + elif is_extension_array_dtype(obj): + # for pandas extension dtypes assume length 1 arrays / series are scalars + return obj.tolist()[0 if len(obj) == 1 else slice(None)] elif isinstance(obj, set): return list(obj) - elif isinstance(obj, type(pd.NaT)): + elif isinstance(obj, (type(pd.NaT), type(pd.NA))): return None # when logging a series directly, numpy datatypes are used elif isinstance(obj, np.datetime64): diff --git a/src/tlo/logging/helpers.py b/src/tlo/logging/helpers.py index 2195c602d0..99fc51c473 100644 --- a/src/tlo/logging/helpers.py +++ b/src/tlo/logging/helpers.py @@ -1,26 +1,14 @@ import logging as _logging -import sys -from pathlib import Path -from typing import Dict +from collections.abc import Collection, Iterable +from typing import Dict, List, Optional, Union -from .core import _FORMATTER, _LOGGERS, DEBUG, getLogger +import pandas as pd +from pandas.api.types import is_extension_array_dtype +from .core import getLogger -def set_output_file(log_path: Path) -> _logging.FileHandler: - """Add filehandler to logger - :param log_path: path for file - :return: filehandler object - """ - file_handler = _logging.FileHandler(log_path) - file_handler.setFormatter(_FORMATTER) - getLogger('tlo').handlers = [h for h in getLogger('tlo').handlers - if not isinstance(h, _logging.FileHandler)] - getLogger('tlo').addHandler(file_handler) - return file_handler - - -def set_logging_levels(custom_levels: Dict[str, int]): +def set_logging_levels(custom_levels: Dict[str, int]) -> None: """Set custom logging levels for disease modules :param custom_levels: Dictionary of modules and their level, '*' can be used as a key for all modules @@ -65,23 +53,78 @@ def set_logging_levels(custom_levels: Dict[str, int]): getLogger(logger_name).setLevel(logger_level) -def init_logging(add_stdout_handler=True): - """Initialise default logging with stdout stream""" - for logger_name, logger in _LOGGERS.items(): - logger.reset_attributes() - if add_stdout_handler: - handler = _logging.StreamHandler(sys.stdout) - handler.setLevel(DEBUG) - handler.setFormatter(_FORMATTER) - getLogger('tlo').addHandler(handler) - _logging.basicConfig(level=_logging.WARNING) +def get_dataframe_row_as_dict_for_logging( + dataframe: pd.DataFrame, + row_label: Union[int, str], + columns: Optional[Iterable[str]] = None, +) -> dict: + """Get row of a pandas dataframe in a format suitable for logging. + + Retrieves entries for all or a subset of columns for a particular row in a dataframe + and returns a dict keyed by column name, with values NumPy or pandas extension types + which should be the same for all rows in dataframe. + + :param dataframe: Population properties dataframe to get properties from. + :param row_label: Unique index label identifying row in dataframe. + :param columns: Set of column names to extract - if ``None``, the default, all + column values will be returned. + :returns: Dictionary with column names as keys and corresponding entries in row as + values. + """ + dataframe = dataframe.convert_dtypes(convert_integer=False, convert_floating=False) + columns = dataframe.columns if columns is None else columns + row_index = dataframe.index.get_loc(row_label) + return { + column_name: + dataframe[column_name].values[row_index] + # pandas extension array datatypes such as nullable types and categoricals, will + # be type unstable if a scalar is returned as NA / NaT / NaN entries will have a + # different type from non-missing entries, therefore use a length 1 array of + # relevant NumPy or pandas extension type in these cases to ensure type + # stability across different rows. + if not is_extension_array_dtype(dataframe[column_name].dtype) else + dataframe[column_name].values[row_index:row_index+1] + for column_name in columns + } -def set_simulation(simulation): - """ - Inject simulation into logger for structured logging, called by the simulation - :param simulation: - :return: +def grouped_counts_with_all_combinations( + dataframe: pd.DataFrame, + group_by_columns: List[str], + column_possible_values: Optional[Dict[str, Collection]] = None, +) -> pd.Series: + """Perform group-by count in which all combinations of column values are included. + + As all combinations are included irrespective of whether they are present in data + (and so have a non-zero count), this gives a multi-index series output of fixed + structure suitable for logging. + + Attempts to convert all columns to categorical datatype, with bool(ean) columns + automatically converted, and other non-categorical columns needing to have set of + possible values specified (which requires that this set is finite). + + :param dataframe: Dataframe to perform group-by counts on. + :param group_by_columns: Columns to perform grouping on. + :param column_possible_values: Dictionary mapping from column names to set of + possible values for all columns not of categorical or bool(ean) data type. + :returns: Multi-index series with values corresponding to grouped counts. """ - logger = getLogger('tlo') - logger.simulation = simulation + subset = dataframe[group_by_columns].copy() + # Convert any bool(ean) columns to categoricals + for column_name in group_by_columns: + if subset[column_name].dtype in ("bool", "boolean"): + subset[column_name] = pd.Categorical( + subset[column_name], categories=[True, False] + ) + # For other non-categorical columns possible values need to be explicitly stated + if column_possible_values is not None: + for column_name, possible_values in column_possible_values.items(): + subset[column_name] = pd.Categorical( + subset[column_name], categories=possible_values + ) + if not (subset.dtypes == "category").all(): + msg = "At least one column not convertable to categorical dtype:\n" + str( + {subset.dtypes[subset.dtypes != "categorical"]} + ) + raise ValueError(msg) + return subset.groupby(by=group_by_columns).size() diff --git a/src/tlo/methods/cardio_metabolic_disorders.py b/src/tlo/methods/cardio_metabolic_disorders.py index d90688adb0..3c985c2bf1 100644 --- a/src/tlo/methods/cardio_metabolic_disorders.py +++ b/src/tlo/methods/cardio_metabolic_disorders.py @@ -1306,7 +1306,7 @@ def proportion_of_something_in_a_groupby_ready_for_logging(_df, something, group df.age_years >= 20)]) / len(df[df[f'nc_{condition}'] & df.is_alive & (df.age_years >= 20)]) } else: - diagnosed = {0.0} + diagnosed = {f'{condition}_diagnosis_prevalence': float("nan")} logger.info( key=f'{condition}_diagnosis_prevalence', @@ -1320,7 +1320,7 @@ def proportion_of_something_in_a_groupby_ready_for_logging(_df, something, group df.age_years >= 20)]) / len(df[df[f'nc_{condition}'] & df.is_alive & (df.age_years >= 20)]) } else: - on_medication = {0.0} + on_medication = {f'{condition}_medication_prevalence': float("nan")} logger.info( key=f'{condition}_medication_prevalence', diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py index dba3bcda8e..69ce038299 100644 --- a/src/tlo/methods/care_of_women_during_pregnancy.py +++ b/src/tlo/methods/care_of_women_during_pregnancy.py @@ -490,9 +490,9 @@ def further_on_birth_care_of_women_in_pregnancy(self, mother_id): # We log the total number of ANC contacts a woman has undergone at the time of birth via this dictionary if 'ga_anc_one' in mni[mother_id]: - ga_anc_one = mni[mother_id]['ga_anc_one'] + ga_anc_one = float(mni[mother_id]['ga_anc_one']) else: - ga_anc_one = 0 + ga_anc_one = 0.0 total_anc_visit_count = {'person_id': mother_id, 'total_anc': df.at[mother_id, 'ac_total_anc_visits_current_pregnancy'], diff --git a/src/tlo/methods/consumables.py b/src/tlo/methods/consumables.py index 9a96ae93cd..674035ad98 100644 --- a/src/tlo/methods/consumables.py +++ b/src/tlo/methods/consumables.py @@ -265,15 +265,26 @@ def _lookup_availability_of_consumables(self, return avail def on_simulation_end(self): - """Do tasks at the end of the simulation: Raise warnings and enter to log about item_codes not recognised.""" + """Do tasks at the end of the simulation. + + Raise warnings and enter to log about item_codes not recognised. + """ if self._not_recognised_item_codes: - warnings.warn(UserWarning(f"Item_Codes were not recognised./n" - f"{self._not_recognised_item_codes}")) - for _treatment_id, _item_codes in self._not_recognised_item_codes: - logger.info( - key="item_codes_not_recognised", - data={_treatment_id if _treatment_id is not None else "": list(_item_codes)} + warnings.warn( + UserWarning( + f"Item_Codes were not recognised./n" + f"{self._not_recognised_item_codes}" ) + ) + logger.info( + key="item_codes_not_recognised", + data={ + _treatment_id if _treatment_id is not None else "": list( + _item_codes + ) + for _treatment_id, _item_codes in self._not_recognised_item_codes + }, + ) def on_end_of_year(self): self._summary_counter.write_to_log_and_reset_counters() diff --git a/src/tlo/methods/demography.py b/src/tlo/methods/demography.py index 8d510f29ae..e58f3895f4 100644 --- a/src/tlo/methods/demography.py +++ b/src/tlo/methods/demography.py @@ -26,6 +26,7 @@ logging, ) from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent +from tlo.logging.helpers import get_dataframe_row_as_dict_for_logging from tlo.methods.causes import ( Cause, collect_causes_from_disease_modules, @@ -124,7 +125,6 @@ def __init__(self, name=None, resourcefilepath=None, equal_allocation_by_distric 'date_of_death': Property(Types.DATE, 'Date of death of this individual'), 'sex': Property(Types.CATEGORICAL, 'Male or female', categories=['M', 'F']), 'mother_id': Property(Types.INT, 'Unique identifier of mother of this individual'), - 'district_num_of_residence': Property(Types.INT, 'The district number in which the person is resident'), # the categories of these properties are set in `pre_initialise_population` 'cause_of_death': Property( @@ -133,6 +133,12 @@ def __init__(self, name=None, resourcefilepath=None, equal_allocation_by_distric categories=['SET_AT_RUNTIME'] ), + 'district_num_of_residence': Property( + Types.CATEGORICAL, + 'The district number in which the person is resident', + categories=['SET_AT_RUNTIME'] + ), + 'district_of_residence': Property( Types.CATEGORICAL, 'The district (name) of residence (mapped from district_num_of_residence).', @@ -220,6 +226,11 @@ def pre_initialise_population(self): 'The cause of death of this individual (the tlo_cause defined by the module)', categories=list(self.causes_of_death.keys()) ) + self.PROPERTIES['district_num_of_residence'] = Property( + Types.CATEGORICAL, + 'The district (name) of residence (mapped from district_num_of_residence).', + categories=sorted(self.parameters['district_num_to_region_name']), + ) self.PROPERTIES['district_of_residence'] = Property( Types.CATEGORICAL, 'The district (name) of residence (mapped from district_num_of_residence).', @@ -497,7 +508,7 @@ def do_death(self, individual_id: int, cause: str, originating_module: Module): data_to_log_for_each_death = { 'age': person['age_years'], 'sex': person['sex'], - 'cause': cause, + 'cause': str(cause), 'label': self.causes_of_death[cause].label, 'person_id': individual_id, 'li_wealth': person['li_wealth'] if 'li_wealth' in person else -99, @@ -513,7 +524,7 @@ def do_death(self, individual_id: int, cause: str, originating_module: Module): # - log all the properties for the deceased person logger_detail.info(key='properties_of_deceased_persons', - data=person.to_dict(), + data=get_dataframe_row_as_dict_for_logging(df, individual_id), description='values of all properties at the time of death for deceased persons') # - log the death in the Deviance module (if it is registered) @@ -799,7 +810,7 @@ def apply(self, population): num_children = pd.Series(index=range(5), data=0).add( df[df.is_alive & (df.age_years < 5)].groupby('age_years').size(), fill_value=0 - ) + ).astype(int) logger.info(key='num_children', data=num_children.to_dict()) diff --git a/src/tlo/methods/depression.py b/src/tlo/methods/depression.py index 4e6825cbc5..a0ffdd12b2 100644 --- a/src/tlo/methods/depression.py +++ b/src/tlo/methods/depression.py @@ -869,10 +869,10 @@ def apply(self, population): n_ever_talk_ther = (df.de_ever_talk_ther & df.is_alive & df.de_depr).sum() def zero_out_nan(x): - return x if not np.isnan(x) else 0 + return x if not np.isnan(x) else 0.0 def safe_divide(x, y): - return x / y if y > 0.0 else 0.0 + return float(x / y) if y > 0.0 else 0.0 dict_for_output = { 'prop_ge15_depr': zero_out_nan(safe_divide(n_ge15_depr, n_ge15)), diff --git a/src/tlo/methods/enhanced_lifestyle.py b/src/tlo/methods/enhanced_lifestyle.py index 008424ec2b..26c79d9587 100644 --- a/src/tlo/methods/enhanced_lifestyle.py +++ b/src/tlo/methods/enhanced_lifestyle.py @@ -12,6 +12,7 @@ from tlo.analysis.utils import flatten_multi_index_series_into_dict_for_logging from tlo.events import PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType, Predictor +from tlo.logging.helpers import grouped_counts_with_all_combinations from tlo.util import get_person_id_to_inherit_from logger = logging.getLogger(__name__) @@ -1939,33 +1940,42 @@ def apply(self, population): for _property in all_lm_keys: if _property in log_by_age_15up: if _property in cat_by_rural_urban_props: - data = df.loc[df.is_alive & (df.age_years >= 15)].groupby(by=[ - 'li_urban', 'sex', _property, 'age_range']).size() + data = grouped_counts_with_all_combinations( + df.loc[df.is_alive & (df.age_years >= 15)], + ["li_urban", "sex", _property, "age_range"] + ) else: - data = df.loc[df.is_alive & (df.age_years >= 15)].groupby(by=[ - 'sex', _property, 'age_range']).size() - + data = grouped_counts_with_all_combinations( + df.loc[df.is_alive & (df.age_years >= 15)], + ["sex", _property, "age_range"] + ) elif _property == 'li_in_ed': - data = df.loc[df.is_alive & df.age_years.between(5, 19)].groupby(by=[ - 'sex', 'li_wealth', _property, 'age_years']).size() - + data = grouped_counts_with_all_combinations( + df.loc[df.is_alive & df.age_years.between(5, 19)], + ["sex", "li_wealth", "li_in_ed", "age_years"], + {"age_years": range(5, 20)} + ) elif _property == 'li_ed_lev': - data = df.loc[df.is_alive & df.age_years.between(15, 49)].groupby(by=[ - 'sex', 'li_wealth', _property, 'age_years']).size() - + data = grouped_counts_with_all_combinations( + df.loc[df.is_alive & df.age_years.between(15, 49)], + ["sex", "li_wealth", "li_ed_lev", "age_years"], + {"age_years": range(15, 50)} + ) elif _property == 'li_is_sexworker': - data = df.loc[df.is_alive & (df.age_years.between(15, 49))].groupby(by=[ - 'sex', _property, 'age_range']).size() - + data = grouped_counts_with_all_combinations( + df.loc[df.is_alive & (df.age_years.between(15, 49))], + ["sex", "li_is_sexworker", "age_range"], + ) elif _property in cat_by_rural_urban_props: # log all properties that are also categorised by rural or urban in addition to ex and age groups - data = df.loc[df.is_alive].groupby(by=[ - 'li_urban', 'sex', _property, 'age_range']).size() - + data = grouped_counts_with_all_combinations( + df.loc[df.is_alive], ["li_urban", "sex", _property, "age_range"] + ) else: # log all other remaining properties - data = df.loc[df.is_alive].groupby(by=['sex', _property, 'age_range']).size() - + data = grouped_counts_with_all_combinations( + df.loc[df.is_alive], ["sex", _property, "age_range"] + ) # log data logger.info( key=_property, diff --git a/src/tlo/methods/epilepsy.py b/src/tlo/methods/epilepsy.py index a1650a3889..5645d55e34 100644 --- a/src/tlo/methods/epilepsy.py +++ b/src/tlo/methods/epilepsy.py @@ -563,16 +563,16 @@ def apply(self, population): n_seiz_stat_1_3 = sum(status_groups.iloc[1:].is_alive) n_seiz_stat_2_3 = sum(status_groups.iloc[2:].is_alive) - n_antiep = (df.is_alive & df.ep_antiep).sum() + n_antiep = int((df.is_alive & df.ep_antiep).sum()) - n_epi_death = df.ep_epi_death.sum() + n_epi_death = int(df.ep_epi_death.sum()) status_groups['prop_seiz_stats'] = status_groups.is_alive / sum(status_groups.is_alive) status_groups['prop_seiz_stat_on_anti_ep'] = status_groups['ep_antiep'] / status_groups.is_alive status_groups['prop_seiz_stat_on_anti_ep'] = status_groups['prop_seiz_stat_on_anti_ep'].fillna(0) epi_death_rate = \ - (n_epi_death * 4 * 1000) / n_seiz_stat_2_3 if n_seiz_stat_2_3 > 0 else 0 + (n_epi_death * 4 * 1000) / n_seiz_stat_2_3 if n_seiz_stat_2_3 > 0 else 0.0 cum_deaths = (~df.is_alive).sum() diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py index e00bf030fd..bf0d6fc0ae 100644 --- a/src/tlo/methods/equipment.py +++ b/src/tlo/methods/equipment.py @@ -6,6 +6,7 @@ import pandas as pd from tlo import logging +from tlo.logging.helpers import get_dataframe_row_as_dict_for_logging logger_summary = logging.getLogger("tlo.methods.healthsystem.summary") @@ -239,14 +240,13 @@ def set_of_keys_or_empty_set(x: Union[set, dict]): right_index=True, how='left', ).drop(columns=['Facility_ID', 'Facility_Name']) - # Log multi-row data-frame - for _, row in output.iterrows(): + for row_index in output.index: logger_summary.info( key='EquipmentEverUsed_ByFacilityID', description='For each facility_id (the set of facilities of the same level in a district), the set of' 'equipment items that are ever used.', - data=row.to_dict(), + data=get_dataframe_row_as_dict_for_logging(output, row_index) ) def from_pkg_names(self, pkg_names: Union[str, Iterable[str]]) -> Set[int]: diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py index db9173d15b..cf51b4c0ab 100644 --- a/src/tlo/methods/healthsystem.py +++ b/src/tlo/methods/healthsystem.py @@ -1208,8 +1208,13 @@ def load_priority_policy(self, policy): ].iloc[0] # Convert policy dataframe into dictionary to speed-up look-up process. - self.priority_rank_dict = \ - Policy_df.set_index("Treatment", drop=True).to_dict(orient="index") + self.priority_rank_dict = ( + Policy_df.set_index("Treatment", drop=True) + # Standardize dtypes to ensure any integers represented as floats are + # converted to integer dtypes + .convert_dtypes() + .to_dict(orient="index") + ) del self.priority_rank_dict["lowest_priority_considered"] def schedule_hsi_event( @@ -1783,7 +1788,7 @@ def write_to_never_ran_hsi_log( 'Number_By_Appt_Type_Code': dict(event_details.appt_footprint), 'Person_ID': person_id, 'priority': priority, - 'Facility_Level': event_details.facility_level if event_details.facility_level is not None else -99, + 'Facility_Level': event_details.facility_level if event_details.facility_level is not None else "-99", 'Facility_ID': facility_id if facility_id is not None else -99, }, description="record of each HSI event that never ran" diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py index 1ddafe4c47..4c4e5d9c14 100644 --- a/src/tlo/methods/hiv.py +++ b/src/tlo/methods/hiv.py @@ -3339,15 +3339,15 @@ def treatment_counts(subset): count = sum(subset) # proportion of subset living with HIV that are diagnosed: proportion_diagnosed = ( - sum(subset & df.hv_diagnosed) / count if count > 0 else 0 + sum(subset & df.hv_diagnosed) / count if count > 0 else 0.0 ) # proportions of subset living with HIV on treatment: art = sum(subset & (df.hv_art != "not")) - art_cov = art / count if count > 0 else 0 + art_cov = art / count if count > 0 else 0.0 # proportion of subset on treatment that have good VL suppression art_vs = sum(subset & (df.hv_art == "on_VL_suppressed")) - art_cov_vs = art_vs / art if art > 0 else 0 + art_cov_vs = art_vs / art if art > 0 else 0.0 return proportion_diagnosed, art_cov, art_cov_vs alive_infected = df.is_alive & df.hv_inf diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py index 695dbeb501..35081b7d27 100644 --- a/src/tlo/methods/labour.py +++ b/src/tlo/methods/labour.py @@ -10,6 +10,7 @@ from tlo import Date, DateOffset, Module, Parameter, Property, Types, logging from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent from tlo.lm import LinearModel, LinearModelType +from tlo.logging.helpers import get_dataframe_row_as_dict_for_logging from tlo.methods import Metadata, labour_lm, pregnancy_helper_functions from tlo.methods.causes import Cause from tlo.methods.dxmanager import DxTest @@ -1056,7 +1057,7 @@ def further_on_birth_labour(self, mother_id): # log delivery setting logger.info(key='delivery_setting_and_mode', data={'mother': mother_id, - 'facility_type': mni[mother_id]['delivery_setting'], + 'facility_type': str(mni[mother_id]['delivery_setting']), 'mode': mni[mother_id]['mode_of_delivery']}) # Store only live births to a mother parity @@ -2611,7 +2612,7 @@ def apply(self, individual_id): self.module.set_intrapartum_complications(individual_id, complication=complication) if df.at[individual_id, 'la_obstructed_labour']: - logger.info(key='maternal_complication', data={'mother': individual_id, + logger.info(key='maternal_complication', data={'person': individual_id, 'type': 'obstructed_labour', 'timing': 'intrapartum'}) @@ -2976,7 +2977,7 @@ def apply(self, person_id, squeeze_factor): self.module.progression_of_hypertensive_disorders(person_id, property_prefix='ps') if df.at[person_id, 'la_obstructed_labour']: - logger.info(key='maternal_complication', data={'mother': person_id, + logger.info(key='maternal_complication', data={'person': person_id, 'type': 'obstructed_labour', 'timing': 'intrapartum'}) @@ -3117,7 +3118,7 @@ def apply(self, person_id, squeeze_factor): # log the PNC visit logger.info(key='postnatal_check', data={'person_id': person_id, - 'delivery_setting': mni[person_id]['delivery_setting'], + 'delivery_setting': str(mni[person_id]['delivery_setting']), 'visit_number': df.at[person_id, 'la_pn_checks_maternal'], 'timing': mni[person_id]['will_receive_pnc']}) @@ -3253,8 +3254,10 @@ def apply(self, person_id, squeeze_factor): # If intervention is delivered - add used equipment self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery')) - person = df.loc[person_id] - logger.info(key='caesarean_delivery', data=person.to_dict()) + logger.info( + key='caesarean_delivery', + data=get_dataframe_row_as_dict_for_logging(df, person_id), + ) logger.info(key='cs_indications', data={'id': person_id, 'indication': mni[person_id]['cs_indication']}) diff --git a/src/tlo/methods/malaria.py b/src/tlo/methods/malaria.py index 3e273245eb..8c451b62d0 100644 --- a/src/tlo/methods/malaria.py +++ b/src/tlo/methods/malaria.py @@ -770,14 +770,14 @@ def check_if_fever_is_caused_by_malaria( # Log the test: line-list of summary information about each test logger.info( key="rdt_log", - data={ - "person_id": person_id, - "age": patient_age, - "fever_present": fever_is_a_symptom, - "rdt_result": dx_result, - "facility_level": facility_level, - "called_by": treatment_id, - }, + data=_data_for_rdt_log( + person_id=person_id, + age=patient_age, + fever_is_a_symptom=fever_is_a_symptom, + dx_result=dx_result, + facility_level=facility_level, + treatment_id=treatment_id + ) ) # Severe malaria infection always returns positive RDT @@ -1061,14 +1061,14 @@ def apply(self, person_id, squeeze_factor): # Log the test: line-list of summary information about each test fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id=person_id) - person_details_for_test = { - 'person_id': person_id, - 'age': df.at[person_id, 'age_years'], - 'fever_present': fever_present, - 'rdt_result': dx_result, - 'facility_level': self.ACCEPTED_FACILITY_LEVEL, - 'called_by': self.TREATMENT_ID - } + person_details_for_test = _data_for_rdt_log( + person_id=person_id, + age=df.at[person_id, 'age_years'], + fever_is_a_symptom=fever_present, + dx_result=dx_result, + facility_level=self.ACCEPTED_FACILITY_LEVEL, + treatment_id=self.TREATMENT_ID, + ) logger.info(key='rdt_log', data=person_details_for_test) if dx_result: @@ -1153,14 +1153,15 @@ def apply(self, person_id, squeeze_factor): # Log the test: line-list of summary information about each test fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id=person_id) - person_details_for_test = { - 'person_id': person_id, - 'age': df.at[person_id, 'age_years'], - 'fever_present': fever_present, - 'rdt_result': dx_result, - 'facility_level': self.ACCEPTED_FACILITY_LEVEL, - 'called_by': self.TREATMENT_ID - } + person_details_for_test = _data_for_rdt_log( + person_id=person_id, + age=df.at[person_id, 'age_years'], + fever_is_a_symptom=fever_present, + dx_result=dx_result, + facility_level=self.ACCEPTED_FACILITY_LEVEL, + treatment_id=self.TREATMENT_ID, + ) + logger.info(key='rdt_log', data=person_details_for_test) # if positive, refer for a confirmatory test at level 1a @@ -1215,14 +1216,14 @@ def apply(self, person_id, squeeze_factor): # rdt is offered as part of the treatment package # Log the test: line-list of summary information about each test fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id=person_id) - person_details_for_test = { - 'person_id': person_id, - 'age': df.at[person_id, 'age_years'], - 'fever_present': fever_present, - 'rdt_result': True, - 'facility_level': self.ACCEPTED_FACILITY_LEVEL, - 'called_by': self.TREATMENT_ID - } + person_details_for_test = _data_for_rdt_log( + person_id=person_id, + age=df.at[person_id, 'age_years'], + fever_is_a_symptom=fever_present, + dx_result=True, + facility_level=self.ACCEPTED_FACILITY_LEVEL, + treatment_id=self.TREATMENT_ID, + ) logger.info(key='rdt_log', data=person_details_for_test) def get_drugs(self, age_of_person): @@ -1312,14 +1313,14 @@ def apply(self, person_id, squeeze_factor): # rdt is offered as part of the treatment package # Log the test: line-list of summary information about each test fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id=person_id) - person_details_for_test = { - 'person_id': person_id, - 'age': df.at[person_id, 'age_years'], - 'fever_present': fever_present, - 'rdt_result': True, - 'facility_level': self.ACCEPTED_FACILITY_LEVEL, - 'called_by': self.TREATMENT_ID - } + person_details_for_test = _data_for_rdt_log( + person_id=person_id, + age=df.at[person_id, 'age_years'], + fever_is_a_symptom=fever_present, + dx_result=True, + facility_level=self.ACCEPTED_FACILITY_LEVEL, + treatment_id=self.TREATMENT_ID, + ) logger.info(key='rdt_log', data=person_details_for_test) def did_not_run(self): @@ -1756,3 +1757,21 @@ def apply(self, population): logger.info(key='pop_district', data=pop.to_dict(), description='District population sizes') + + +def _data_for_rdt_log( + person_id: int, + age: int, + fever_is_a_symptom: bool, + dx_result: Union[bool, None], + facility_level: str, + treatment_id: str, +): + return { + "person_id": person_id, + "age": age, + "fever_present": fever_is_a_symptom, + "rdt_result": pd.array([dx_result], dtype="boolean"), + "facility_level": facility_level, + "called_by": treatment_id, + } diff --git a/src/tlo/methods/measles.py b/src/tlo/methods/measles.py index 5d2c6dcc53..39f9828860 100644 --- a/src/tlo/methods/measles.py +++ b/src/tlo/methods/measles.py @@ -548,7 +548,7 @@ def apply(self, population): if tmp: proportion_with_symptom = number_with_symptom / tmp else: - proportion_with_symptom = 0 + proportion_with_symptom = 0.0 symptom_output[symptom] = proportion_with_symptom logger.info(key="measles_symptoms", @@ -586,7 +586,7 @@ def apply(self, population): if total_infected: prop_infected_by_age = infected_age_counts / total_infected else: - prop_infected_by_age = infected_age_counts # just output the series of zeros by age group + prop_infected_by_age = infected_age_counts.astype("float") # just output the series of zeros by age group logger.info(key='measles_incidence_age_range', data=prop_infected_by_age.to_dict(), description="measles incidence by age group") diff --git a/src/tlo/methods/newborn_outcomes.py b/src/tlo/methods/newborn_outcomes.py index 433b21ca88..3691bc6003 100644 --- a/src/tlo/methods/newborn_outcomes.py +++ b/src/tlo/methods/newborn_outcomes.py @@ -1363,7 +1363,7 @@ def apply(self, person_id, squeeze_factor): # Log the PNC check logger.info(key='postnatal_check', data={'person_id': person_id, - 'delivery_setting': nci[person_id]['delivery_setting'], + 'delivery_setting': str(nci[person_id]['delivery_setting']), 'visit_number': df.at[person_id, 'nb_pnc_check'], 'timing': nci[person_id]['will_receive_pnc']}) diff --git a/src/tlo/methods/rti.py b/src/tlo/methods/rti.py index 13ddee6a86..17de0b0451 100644 --- a/src/tlo/methods/rti.py +++ b/src/tlo/methods/rti.py @@ -2447,7 +2447,7 @@ def rti_assign_injuries(self, number): inc_other = other_counts / ((n_alive - other_counts) * 1 / 12) * 100000 tot_inc_all_inj = inc_amputations + inc_burns + inc_fractures + inc_tbi + inc_sci + inc_minor + inc_other if number > 0: - number_of_injuries = inj_df['Number_of_injuries'].tolist() + number_of_injuries = int(inj_df['Number_of_injuries'].iloc[0]) else: number_of_injuries = 0 dict_to_output = {'inc_amputations': inc_amputations, @@ -2489,7 +2489,7 @@ def rti_assign_injuries(self, number): if n_lx_fracs > 0: proportion_lx_fracture_open = n_open_lx_fracs / n_lx_fracs else: - proportion_lx_fracture_open = 'no_lx_fractures' + proportion_lx_fracture_open = float("nan") injury_info = {'Proportion_lx_fracture_open': proportion_lx_fracture_open} logger.info(key='Open_fracture_information', data=injury_info, @@ -2814,7 +2814,7 @@ def apply(self, population): df.loc[shock_index, 'rt_in_shock'] = True # log the percentage of those with RTIs in shock percent_in_shock = \ - len(shock_index) / len(selected_for_rti_inj) if len(selected_for_rti_inj) > 0 else 'none_injured' + len(shock_index) / len(selected_for_rti_inj) if len(selected_for_rti_inj) > 0 else float("nan") logger.info(key='Percent_of_shock_in_rti', data={'Percent_of_shock_in_rti': percent_in_shock}, description='The percentage of those assigned injuries who were also assign the shock property') @@ -5572,7 +5572,7 @@ def apply(self, population): label: ( len(pop_subset.loc[pop_subset['rt_inj_severity'] == 'severe']) / len(pop_subset) - ) if len(pop_subset) > 0 else "none_injured" + ) if len(pop_subset) > 0 else float("nan") for label, pop_subset in population_subsets_with_injuries.items() } self.totmild += (population_with_injuries.rt_inj_severity == "mild").sum() @@ -5588,25 +5588,25 @@ def apply(self, population): description='severity of injuries in simulation') # ==================================== Incidence ============================================================== # How many were involved in a RTI - n_in_RTI = df.rt_road_traffic_inc.sum() + n_in_RTI = int(df.rt_road_traffic_inc.sum()) children_in_RTI = len(df.loc[df.rt_road_traffic_inc & (df['age_years'] < 19)]) children_alive = len(df.loc[df['age_years'] < 19]) self.numerator += n_in_RTI self.totinjured += n_in_RTI # How many were disabled - n_perm_disabled = (df.is_alive & df.rt_perm_disability).sum() + n_perm_disabled = int((df.is_alive & df.rt_perm_disability).sum()) # self.permdis += n_perm_disabled - n_alive = df.is_alive.sum() + n_alive = int(df.is_alive.sum()) self.denominator += (n_alive - n_in_RTI) * (1 / 12) - n_immediate_death = (df.rt_road_traffic_inc & df.rt_imm_death).sum() + n_immediate_death = int((df.rt_road_traffic_inc & df.rt_imm_death).sum()) self.deathonscene += n_immediate_death diedfromrtiidx = df.index[df.rt_imm_death | df.rt_post_med_death | df.rt_no_med_death | df.rt_death_from_shock | df.rt_unavailable_med_death] - n_sought_care = (df.rt_road_traffic_inc & df.rt_med_int).sum() + n_sought_care = int((df.rt_road_traffic_inc & df.rt_med_int).sum()) self.soughtmedcare += n_sought_care - n_death_post_med = df.rt_post_med_death.sum() + n_death_post_med = int(df.rt_post_med_death.sum()) self.deathaftermed += n_death_post_med - self.deathwithoutmed += df.rt_no_med_death.sum() + self.deathwithoutmed += int(df.rt_no_med_death.sum()) self.death_inc_numerator += n_immediate_death + n_death_post_med + len(df.loc[df.rt_no_med_death]) self.death_in_denominator += (n_alive - (n_immediate_death + n_death_post_med + len(df.loc[df.rt_no_med_death]) )) * \ @@ -5615,7 +5615,7 @@ def apply(self, population): percent_accidents_result_in_death = \ (self.deathonscene + self.deathaftermed + self.deathwithoutmed) / self.numerator else: - percent_accidents_result_in_death = 'none injured' + percent_accidents_result_in_death = float("nan") maleinrti = len(df.loc[df.rt_road_traffic_inc & (df['sex'] == 'M')]) femaleinrti = len(df.loc[df.rt_road_traffic_inc & (df['sex'] == 'F')]) @@ -5624,35 +5624,35 @@ def apply(self, population): maleinrti = maleinrti / divider femaleinrti = femaleinrti / divider else: - maleinrti = 1 - femaleinrti = 0 + maleinrti = 1.0 + femaleinrti = 0.0 mfratio = [maleinrti, femaleinrti] if (n_in_RTI - len(df.loc[df.rt_imm_death])) > 0: percent_sought_care = n_sought_care / (n_in_RTI - len(df.loc[df.rt_imm_death])) else: - percent_sought_care = 'none_injured' + percent_sought_care = float("nan") if n_sought_care > 0: percent_died_post_care = n_death_post_med / n_sought_care else: - percent_died_post_care = 'none_injured' + percent_died_post_care = float("nan") if n_sought_care > 0: percentage_admitted_to_ICU_or_HDU = len(df.loc[df.rt_med_int & df.rt_in_icu_or_hdu]) / n_sought_care else: - percentage_admitted_to_ICU_or_HDU = 'none_injured' + percentage_admitted_to_ICU_or_HDU = float("nan") if (n_alive - n_in_RTI) > 0: inc_rti = (n_in_RTI / ((n_alive - n_in_RTI) * (1 / 12))) * 100000 else: - inc_rti = 0 + inc_rti = 0.0 if (children_alive - children_in_RTI) > 0: inc_rti_in_children = (children_in_RTI / ((children_alive - children_in_RTI) * (1 / 12))) * 100000 else: - inc_rti_in_children = 0 + inc_rti_in_children = 0.0 if (n_alive - len(diedfromrtiidx)) > 0: inc_rti_death = (len(diedfromrtiidx) / ((n_alive - len(diedfromrtiidx)) * (1 / 12))) * 100000 else: - inc_rti_death = 0 + inc_rti_death = 0.0 if (n_alive - len(df.loc[df.rt_post_med_death])) > 0: inc_post_med_death = (len(df.loc[df.rt_post_med_death]) / ((n_alive - len(df.loc[df.rt_post_med_death])) * (1 / 12))) * 100000 @@ -5662,21 +5662,21 @@ def apply(self, population): inc_imm_death = (len(df.loc[df.rt_imm_death]) / ((n_alive - len(df.loc[df.rt_imm_death])) * (1 / 12))) * \ 100000 else: - inc_imm_death = 0 + inc_imm_death = 0.0 if (n_alive - len(df.loc[df.rt_no_med_death])) > 0: inc_death_no_med = (len(df.loc[df.rt_no_med_death]) / ((n_alive - len(df.loc[df.rt_no_med_death])) * (1 / 12))) * 100000 else: - inc_death_no_med = 0 + inc_death_no_med = 0.0 if (n_alive - len(df.loc[df.rt_unavailable_med_death])) > 0: inc_death_unavailable_med = (len(df.loc[df.rt_unavailable_med_death]) / ((n_alive - len(df.loc[df.rt_unavailable_med_death])) * (1 / 12))) * 100000 else: - inc_death_unavailable_med = 0 + inc_death_unavailable_med = 0.0 if self.fracdenominator > 0: frac_incidence = (self.totfracnumber / self.fracdenominator) * 100000 else: - frac_incidence = 0 + frac_incidence = 0.0 # calculate case fatality ratio for those injured who don't seek healthcare did_not_seek_healthcare = len(df.loc[df.rt_road_traffic_inc & ~df.rt_med_int & ~df.rt_diagnosed]) died_no_healthcare = \ @@ -5684,12 +5684,12 @@ def apply(self, population): if did_not_seek_healthcare > 0: cfr_no_med = died_no_healthcare / did_not_seek_healthcare else: - cfr_no_med = 'all_sought_care' + cfr_no_med = float("nan") # calculate incidence rate per 100,000 of deaths on scene if n_alive > 0: inc_death_on_scene = (len(df.loc[df.rt_imm_death]) / n_alive) * 100000 * (1 / 12) else: - inc_death_on_scene = 0 + inc_death_on_scene = 0.0 dict_to_output = { 'number involved in a rti': n_in_RTI, 'incidence of rti per 100,000': inc_rti, @@ -5727,7 +5727,7 @@ def apply(self, population): percent_related_to_alcohol = len(injuredDemographics.loc[injuredDemographics.li_ex_alc]) / \ len(injuredDemographics) except ZeroDivisionError: - percent_related_to_alcohol = 0 + percent_related_to_alcohol = 0.0 injured_demography_summary = { 'males_in_rti': injuredDemographics['sex'].value_counts()['M'], 'females_in_rti': injuredDemographics['sex'].value_counts()['F'], diff --git a/src/tlo/methods/stunting.py b/src/tlo/methods/stunting.py index 002d24bc31..ec2725bd39 100644 --- a/src/tlo/methods/stunting.py +++ b/src/tlo/methods/stunting.py @@ -524,7 +524,9 @@ def apply(self, population): """Log the current distribution of stunting classification by age""" df = population.props - d_to_log = df.loc[df.is_alive & (df.age_years < 5)].groupby( + subset = df.loc[df.is_alive & (df.age_years < 5)].copy() + subset["age_years"] = pd.Categorical(subset["age_years"], categories=range(5)) + d_to_log = subset.groupby( by=['age_years', 'un_HAZ_category']).size().sort_index().to_dict() def convert_keys_to_string(d): diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py index 8b7a061586..5f87bcd261 100644 --- a/src/tlo/methods/tb.py +++ b/src/tlo/methods/tb.py @@ -2719,7 +2719,7 @@ def apply(self, population): ) # proportion of active TB cases in the last year who are HIV-positive - prop_hiv = inc_active_hiv / new_tb_cases if new_tb_cases else 0 + prop_hiv = inc_active_hiv / new_tb_cases if new_tb_cases else 0.0 logger.info( key="tb_incidence", @@ -2753,7 +2753,7 @@ def apply(self, population): df[(df.age_years >= 15) & df.is_alive] ) if len( df[(df.age_years >= 15) & df.is_alive] - ) else 0 + ) else 0.0 assert prev_active_adult <= 1 # prevalence of active TB in children @@ -2764,7 +2764,7 @@ def apply(self, population): df[(df.age_years < 15) & df.is_alive] ) if len( df[(df.age_years < 15) & df.is_alive] - ) else 0 + ) else 0.0 assert prev_active_child <= 1 # LATENT @@ -2781,7 +2781,7 @@ def apply(self, population): df[(df.age_years >= 15) & df.is_alive] ) if len( df[(df.age_years >= 15) & df.is_alive] - ) else 0 + ) else 0.0 assert prev_latent_adult <= 1 # proportion of population with latent TB - children @@ -2823,7 +2823,7 @@ def apply(self, population): if new_mdr_cases: prop_mdr = new_mdr_cases / new_tb_cases else: - prop_mdr = 0 + prop_mdr = 0.0 logger.info( key="tb_mdr", @@ -2845,7 +2845,7 @@ def apply(self, population): if new_tb_diagnosis: prop_dx = new_tb_diagnosis / new_tb_cases else: - prop_dx = 0 + prop_dx = 0.0 # ------------------------------------ TREATMENT ------------------------------------ # number of tb cases who became active in last timeperiod and initiated treatment @@ -2861,7 +2861,7 @@ def apply(self, population): tx_coverage = new_tb_tx / new_tb_cases # assert tx_coverage <= 1 else: - tx_coverage = 0 + tx_coverage = 0.0 # ipt coverage new_tb_ipt = len( @@ -2874,7 +2874,7 @@ def apply(self, population): if new_tb_ipt: current_ipt_coverage = new_tb_ipt / len(df[df.is_alive]) else: - current_ipt_coverage = 0 + current_ipt_coverage = 0.0 logger.info( key="tb_treatment", @@ -2945,7 +2945,7 @@ def apply(self, population): if adult_num_false_positive: adult_prop_false_positive = adult_num_false_positive / new_tb_tx_adult else: - adult_prop_false_positive = 0 + adult_prop_false_positive = 0.0 # children child_num_false_positive = len( diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py index 761c161799..1853c76063 100644 --- a/src/tlo/simulation.py +++ b/src/tlo/simulation.py @@ -68,8 +68,8 @@ def __init__(self, *, start_date: Date, seed: int = None, log_config: dict = Non if log_config is None: log_config = {} self._custom_log_levels = None - self._log_filepath = None - self._configure_logging(**log_config) + self._log_filepath = self._configure_logging(**log_config) + # random number generator seed_from = 'auto' if seed is None else 'user' @@ -99,8 +99,10 @@ def _configure_logging(self, filename: str = None, directory: Union[Path, str] = # clear logging environment # if using progress bar we do not print log messages to stdout to avoid # clashes between progress bar and log output - logging.init_logging(add_stdout_handler=not (self.show_progress_bar or suppress_stdout)) - logging.set_simulation(self) + logging.initialise( + add_stdout_handler=not (self.show_progress_bar or suppress_stdout), + simulation_date_getter=lambda: self.date.isoformat(), + ) if custom_levels: # if modules have already been registered @@ -115,7 +117,6 @@ def _configure_logging(self, filename: str = None, directory: Union[Path, str] = log_path = Path(directory) / f"{filename}__{timestamp}.log" self.output_file = logging.set_output_file(log_path) logger.info(key='info', data=f'Log output: {log_path}') - self._log_filepath = log_path return log_path return None diff --git a/tests/test_logging.py b/tests/test_logging.py index 13151c8be5..6d094623c4 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -1,173 +1,587 @@ +import contextlib import json -import os +import logging as _logging +import sys +from collections.abc import Generator, Iterable, Mapping +from itertools import chain, product, repeat from pathlib import Path +from typing import Callable +import numpy as np import pandas as pd import pytest -from tlo import Date, Simulation, logging -from tlo.methods import demography, enhanced_lifestyle - -start_date = Date(2010, 1, 1) -popsize = 500 - - -@pytest.fixture(scope='function') -def basic_configuration(tmpdir): - """Setup basic file handler configuration""" - # tlo module config - file_name = tmpdir.join('test.log') - file_handler = logging.set_output_file(file_name) - - yield file_handler, file_name - - file_handler.close() - - -@pytest.fixture(scope='function') -def simulation_configuration(tmpdir): - resourcefilepath = Path(os.path.dirname(__file__)) / '../resources' - - sim = Simulation(start_date=start_date, log_config={'filename': 'log', 'directory': tmpdir}) - sim.register(demography.Demography(resourcefilepath=resourcefilepath), - enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath)) - - yield sim.output_file, sim.log_filepath - - sim.output_file.close() - - -def read_file(file_handler, file_name): - """ - Reads file and returns the lines - :param file_handler: filehandler (to flush) though might be a bit unnecessary - :param file_name: path to file - :return: list of lines - """ - file_handler.flush() - with open(file_name) as handle: - lines = handle.readlines() - return lines - - -def log_message(message_level, logger_level, message, logger_name='tlo.test.logger', structured_logging=False): - """ - Sets up logger level, and writes message at the message level - - :param message_level: level that the message will be added as - :param logger_level: level that the logger is set to - :param message: message to be written to log - :param structured_logging: - - """ +import tlo.logging as logging +import tlo.logging.core as core + + +def _single_row_dataframe(data: dict) -> pd.DataFrame: + # Single row dataframe 'type' which allows construction by calling on a dictionary + # of scalars by using an explicit length 1 index while also giving a readable + # test parameter identifier + return pd.DataFrame(data, index=[0]) + + +LOGGING_LEVELS = [logging.DEBUG, logging.INFO, logging.WARNING, logging.CRITICAL] +CATCH_ALL_LEVEL = -1 +STRING_DATA_VALUES = ["foo", "bar", "spam"] +ITERABLE_DATA_VALUES = [(1, 2), (3, 1, 2), ("d", "e"), ("a", "c", 1)] +MAPPING_DATA_VALUES = [{"a": 1, "b": "spam", 2: None}, {"eggs": "foo", "bar": 1.25}] +SUPPORTED_SEQUENCE_TYPES = [list, tuple, pd.Series] +SUPPORTED_ITERABLE_TYPES = SUPPORTED_SEQUENCE_TYPES + [set] +SUPPORTED_MAPPING_TYPES = [dict, _single_row_dataframe] +LOGGER_NAMES = ["tlo", "tlo.methods"] +SIMULATION_DATE = "2010-01-01T00:00:00" + + +class UpdateableSimulateDateGetter: + + def __init__(self, start_date=pd.Timestamp(2010, 1, 1)): + self._date = start_date + + def increment_date(self, days=1) -> None: + self._date += pd.DateOffset(days=days) + + def __call__(self) -> str: + return self._date.isoformat() + + +@pytest.fixture +def simulation_date_getter() -> core.SimulationDateGetter: + return lambda: SIMULATION_DATE + + +@pytest.fixture +def root_level() -> core.LogLevel: + return logging.WARNING + + +@pytest.fixture +def stdout_handler_level() -> core.LogLevel: + return logging.DEBUG + + +@pytest.fixture +def add_stdout_handler() -> bool: + return False + + +@pytest.fixture(autouse=True) +def initialise_logging( + add_stdout_handler: bool, + simulation_date_getter: core.SimulationDateGetter, + root_level: core.LogLevel, + stdout_handler_level: core.LogLevel, +) -> Generator[None, None, None]: + logging.initialise( + add_stdout_handler=add_stdout_handler, + simulation_date_getter=simulation_date_getter, + root_level=root_level, + stdout_handler_level=stdout_handler_level, + ) + yield + logging.reset() + + +@pytest.mark.parametrize("add_stdout_handler", [True, False]) +@pytest.mark.parametrize("root_level", LOGGING_LEVELS, ids=_logging.getLevelName) +@pytest.mark.parametrize( + "stdout_handler_level", LOGGING_LEVELS, ids=_logging.getLevelName +) +def test_initialise_logging( + add_stdout_handler: bool, + simulation_date_getter: core.SimulationDateGetter, + root_level: core.LogLevel, + stdout_handler_level: core.LogLevel, +) -> None: + logger = logging.getLogger("tlo") + assert logger.level == root_level + if add_stdout_handler: + assert len(logger.handlers) == 1 + handler = logger.handlers[0] + assert isinstance(handler, _logging.StreamHandler) + assert handler.stream is sys.stdout + assert handler.level == stdout_handler_level + else: + assert len(logger.handlers) == 0 + assert core._get_simulation_date is simulation_date_getter + + +def _check_handlers( + logger: core.Logger, expected_number_handlers: int, expected_log_path: Path +) -> None: + assert len(logger.handlers) == expected_number_handlers + file_handlers = [h for h in logger.handlers if isinstance(h, _logging.FileHandler)] + assert len(file_handlers) == 1 + assert file_handlers[0].baseFilename == str(expected_log_path) + + +@pytest.mark.parametrize("add_stdout_handler", [True, False]) +def test_set_output_file(add_stdout_handler: bool, tmp_path: Path) -> None: + log_path_1 = tmp_path / "test-1.log" + log_path_2 = tmp_path / "test-2.log" + logging.set_output_file(log_path_1) + logger = logging.getLogger("tlo") + expected_number_handlers = 2 if add_stdout_handler else 1 + _check_handlers(logger, expected_number_handlers, log_path_1) + # Setting output file a second time should replace previous file handler rather + # than add an additional handler and keep existing + logging.set_output_file(log_path_2) + _check_handlers(logger, expected_number_handlers, log_path_2) + + +@pytest.mark.parametrize("logger_name", ["tlo", "tlo.methods"]) +def test_getLogger(logger_name: str) -> None: + logger = logging.getLogger(logger_name) + assert logger.name == logger_name + assert isinstance(logger.handlers, list) + assert isinstance(logger.level, int) + assert logger.isEnabledFor(logger.level) + assert logging.getLogger(logger_name) is logger + + +@pytest.mark.parametrize("logger_name", ["foo", "spam.tlo"]) +def test_getLogger_invalid_name_raises(logger_name: str) -> None: + with pytest.raises(AssertionError, match=logger_name): + logging.getLogger(logger_name) + + +@pytest.mark.parametrize("mapping_data", MAPPING_DATA_VALUES) +@pytest.mark.parametrize("mapping_type", SUPPORTED_MAPPING_TYPES) +def test_get_log_data_as_dict_with_mapping_types( + mapping_data: Mapping, mapping_type: Callable +) -> None: + log_data = mapping_type(mapping_data) + data_dict = core._get_log_data_as_dict(log_data) + assert len(data_dict) == len(mapping_data) + assert set(data_dict.keys()) == set(map(str, mapping_data.keys())) + assert set(data_dict.values()) == set(mapping_data.values()) + # Dictionary returned should be invariant to original ordering + assert data_dict == core._get_log_data_as_dict( + mapping_type(dict(reversed(mapping_data.items()))) + ) + + +@pytest.mark.parametrize("mapping_data", MAPPING_DATA_VALUES) +def test_get_log_data_as_dict_with_multirow_dataframe_raises( + mapping_data: Mapping, +) -> None: + log_data = pd.DataFrame(mapping_data, index=[0, 1]) + with pytest.raises(ValueError, match="multirow"): + core._get_log_data_as_dict(log_data) + + +@pytest.mark.parametrize("values", ITERABLE_DATA_VALUES) +@pytest.mark.parametrize("sequence_type", SUPPORTED_SEQUENCE_TYPES) +def test_get_log_data_as_dict_with_sequence_types( + values: Iterable, sequence_type: Callable +) -> None: + log_data = sequence_type(values) + data_dict = core._get_log_data_as_dict(log_data) + assert len(data_dict) == len(log_data) + assert list(data_dict.keys()) == [f"item_{i+1}" for i in range(len(log_data))] + assert list(data_dict.values()) == list(log_data) + + +@pytest.mark.parametrize("values", ITERABLE_DATA_VALUES) +def test_get_log_data_as_dict_with_set(values: Iterable) -> None: + data = set(values) + data_dict = core._get_log_data_as_dict(data) + assert len(data_dict) == len(data) + assert list(data_dict.keys()) == [f"item_{i+1}" for i in range(len(data))] + assert set(data_dict.values()) == data + # Dictionary returned should be invariant to original ordering + assert data_dict == core._get_log_data_as_dict(set(reversed(values))) + + +def test_convert_numpy_scalars_to_python_types() -> None: + data = { + "a": np.int64(1), + "b": np.int32(42), + "c": np.float64(0.5), + "d": np.bool_(True), + } + expected_converted_data = {"a": 1, "b": 42, "c": 0.5, "d": True} + converted_data = core._convert_numpy_scalars_to_python_types(data) + assert converted_data == expected_converted_data + + +def test_get_columns_from_data_dict() -> None: + data = { + "a": 1, + "b": 0.5, + "c": False, + "d": "foo", + "e": pd.Timestamp("2010-01-01"), + } + expected_columns = { + "a": "int", + "b": "float", + "c": "bool", + "d": "str", + "e": "Timestamp", + } + columns = core._get_columns_from_data_dict(data) + assert columns == expected_columns + + +@contextlib.contextmanager +def _propagate_to_root() -> Generator[None, None, None]: + # Enable propagation to root logger to allow pytest capturing to work + root_logger = logging.getLogger("tlo") + root_logger._std_logger.propagate = True + yield + root_logger._std_logger.propagate = False + + +def _setup_caplog_and_get_logger( + caplog: pytest.LogCaptureFixture, logger_name: str, logger_level: core.LogLevel +) -> core.Logger: + caplog.set_level(CATCH_ALL_LEVEL, logger_name) logger = logging.getLogger(logger_name) logger.setLevel(logger_level) - - if structured_logging: - if message_level == 'logging.DEBUG': - logger.debug(key='structured', data=message) - elif message_level == 'logging.INFO': - logger.info(key='structure', data=message) - elif message_level == 'logging.WARNING': - logger.warning(key='structured', data=message) - elif message_level == 'logging.CRITICAL': - logger.critical(key='structured', data=message) + return logger + + +@pytest.mark.parametrize("disable_level", LOGGING_LEVELS, ids=_logging.getLevelName) +@pytest.mark.parametrize("logger_level_offset", [-5, 0, 5]) +@pytest.mark.parametrize("data", STRING_DATA_VALUES) +@pytest.mark.parametrize("logger_name", LOGGER_NAMES) +def test_disable( + disable_level: core.LogLevel, + logger_level_offset: int, + data: str, + logger_name: str, + caplog: pytest.LogCaptureFixture, +) -> None: + logger = _setup_caplog_and_get_logger(caplog, logger_name, CATCH_ALL_LEVEL) + logging.disable(disable_level) + assert not logger.isEnabledFor(disable_level) + message_level = disable_level + logger_level_offset + with _propagate_to_root(): + logger.log(message_level, key="message", data=data) + if message_level > disable_level: + # Message level is above disable level and so should have been captured + assert len(caplog.records) == 1 + assert data in caplog.records[0].msg + else: + # Message level is below disable level and so should not have been captured + assert len(caplog.records) == 0 + + +def _check_captured_log_output_for_levels( + caplog: pytest.LogCaptureFixture, + message_level: core.LogLevel, + logger_level: core.LogLevel, + data: str, +) -> None: + if message_level >= logger_level: + # Message level is at or above logger's level and so should have been captured + assert len(caplog.records) == 1 + assert data in caplog.records[0].msg else: - if message_level == 'logging.DEBUG': - logger.debug(message) - elif message_level == 'logging.INFO': - logger.info(message) - elif message_level == 'logging.WARNING': - logger.warning(message) - elif message_level == 'logging.CRITICAL': - logger.critical(message) - - -class TestStructuredLogging: - @pytest.mark.parametrize("message_level", ["logging.DEBUG", "logging.INFO", "logging.WARNING", "logging.CRITICAL"]) - def test_messages_same_level(self, simulation_configuration, message_level): - # given that messages are at the same level as the logger - logger_level = eval(message_level) - message = {"message": pd.Series([12.5])[0]} - file_handler, file_path = simulation_configuration - log_message(message_level, logger_level, message, structured_logging=True) - - lines = read_file(file_handler, file_path) - header_json = json.loads(lines[5]) - data_json = json.loads(lines[6]) - - # message should be written to log - assert len(lines) == 7 - assert header_json['level'] == message_level.lstrip("logging.") - assert 'message' in header_json['columns'] - assert header_json['columns']['message'] == 'float64' - assert data_json['values'] == [12.5] - - @pytest.mark.parametrize("message_level", ["logging.DEBUG", "logging.INFO", "logging.WARNING", "logging.CRITICAL"]) - def test_messages_higher_level(self, simulation_configuration, message_level): - # given that messages are a higher level than the logger - logger_level = eval(message_level) - 1 - message = {"message": pd.Series([12.5])[0]} - file_handler, file_path = simulation_configuration - log_message(message_level, logger_level, message, structured_logging=True) - - lines = read_file(file_handler, file_path) - header_json = json.loads(lines[5]) - data_json = json.loads(lines[6]) - - # message should be written to log - assert len(lines) == 7 - assert header_json['level'] == message_level.lstrip("logging.") - assert 'message' in header_json['columns'] - assert header_json['columns']['message'] == 'float64' - assert data_json['values'] == [12.5] - - @pytest.mark.parametrize("message_level", ["logging.DEBUG", "logging.INFO", "logging.WARNING", "logging.CRITICAL"]) - def test_messages_lower_level(self, simulation_configuration, message_level): - # given that messages are at a lower level than logger - logger_level = eval(message_level) + 1 - message = {"message": pd.Series([12.5])[0]} - file_handler, file_path = simulation_configuration - log_message(message_level, logger_level, message, structured_logging=True) - - lines = read_file(file_handler, file_path) - - # only simulation info messages should be written to log - assert len(lines) == 5 - - -class TestConvertLogData: - def setup_method(self): - self.expected_output = {'item_1': 1, 'item_2': 2} - self.logger = logging.getLogger('tlo.test.logger') - - @pytest.mark.parametrize("iterable_data", [[1, 2], {1, 2}, (1, 2)]) - def test_convert_iterable_to_dict(self, iterable_data): - output = self.logger._get_data_as_dict(iterable_data) - assert self.expected_output == output - - def test_convert_df_to_dict(self): - df = pd.DataFrame({'item_1': [1], 'item_2': [2]}) - output = self.logger._get_data_as_dict(df) - - assert self.expected_output == output - - def test_string_to_dict(self): - output = self.logger._get_data_as_dict("strings") - assert {'message': 'strings'} == output - - -def test_mixed_logging(): - """Logging with both oldstyle and structured logging should raise an error""" - logger = logging.getLogger('tlo.test.logger') - logger.setLevel(logging.INFO) - with pytest.raises(ValueError): - logger.info("stdlib method") - logger.info(key="structured", data={"key": 10}) - - -@pytest.mark.parametrize("add_stdout_handler", ((True, False))) -def test_init_logging(add_stdout_handler): - logging.init_logging(add_stdout_handler) - logger = logging.getLogger('tlo') - assert len(logger.handlers) == (1 if add_stdout_handler else 0) + # Message level is below logger's set level and so should not have been captured + assert len(caplog.records) == 0 + + +@pytest.mark.parametrize("message_level", LOGGING_LEVELS, ids=_logging.getLevelName) +@pytest.mark.parametrize("logger_level_offset", [-5, 0, 5]) +@pytest.mark.parametrize("data", STRING_DATA_VALUES) +@pytest.mark.parametrize("logger_name", LOGGER_NAMES) +def test_logging_with_log( + message_level: core.LogLevel, + logger_level_offset: int, + data: str, + logger_name: str, + caplog: pytest.LogCaptureFixture, +) -> None: + logger_level = message_level + logger_level_offset + logger = _setup_caplog_and_get_logger(caplog, logger_name, logger_level) + with _propagate_to_root(): + logger.log(level=message_level, key="message", data=data) + _check_captured_log_output_for_levels(caplog, message_level, logger_level, data) + + +@pytest.mark.parametrize("message_level", LOGGING_LEVELS, ids=_logging.getLevelName) +@pytest.mark.parametrize("logger_level_offset", [-5, 0, 5]) +@pytest.mark.parametrize("logger_name", LOGGER_NAMES) +@pytest.mark.parametrize("data", STRING_DATA_VALUES) +def test_logging_with_convenience_methods( + message_level: core.LogLevel, + logger_level_offset: int, + data: str, + logger_name: str, + caplog: pytest.LogCaptureFixture, +) -> None: + logger_level = message_level + logger_level_offset + logger = _setup_caplog_and_get_logger(caplog, logger_name, logger_level) + convenience_method = getattr(logger, _logging.getLevelName(message_level).lower()) + with _propagate_to_root(): + convenience_method(key="message", data=data) + _check_captured_log_output_for_levels(caplog, message_level, logger_level, data) + + +def _check_header( + header: dict[str, str | dict[str, str]], + expected_module: str, + expected_key: str, + expected_level: str, + expected_description: str, + expected_columns: dict[str, str], +) -> None: + assert set(header.keys()) == { + "uuid", + "type", + "module", + "key", + "level", + "columns", + "description", + } + assert isinstance(header["uuid"], str) + assert set(header["uuid"]) <= set("abcdef0123456789") + assert header["type"] == "header" + assert header["module"] == expected_module + assert header["key"] == expected_key + assert header["level"] == expected_level + assert header["description"] == expected_description + assert isinstance(header["columns"], dict) + assert header["columns"] == expected_columns + + +def _check_row( + row: dict[str, str], + logger_level: core.LogLevel, + expected_uuid: str, + expected_date: str, + expected_values: list, + expected_module: str, + expected_key: str, +) -> None: + assert row["uuid"] == expected_uuid + assert row["date"] == expected_date + assert row["values"] == expected_values + if logger_level == logging.DEBUG: + assert row["module"] == expected_module + assert row["key"] == expected_key + + +def _parse_and_check_log_records( + caplog: pytest.LogCaptureFixture, + logger_name: str, + logger_level: core.LogLevel, + message_level: core.LogLevel, + data_dicts: dict, + dates: str, + keys: str, + description: str | None = None, +) -> None: + headers = {} + for record, data_dict, date, key in zip(caplog.records, data_dicts, dates, keys): + message_lines = record.msg.split("\n") + if key not in headers: + # First record for key therefore expect both header and row lines + assert len(message_lines) == 2 + header_line, row_line = message_lines + headers[key] = json.loads(header_line) + _check_header( + header=headers[key], + expected_module=logger_name, + expected_key=key, + expected_level=_logging.getLevelName(logger_level), + expected_description=description, + expected_columns=logging.core._get_columns_from_data_dict(data_dict), + ) + else: + # Subsequent records for key should only have row line + assert len(message_lines) == 1 + row_line = message_lines[0] + row = json.loads(row_line) + _check_row( + row=row, + logger_level=message_level, + expected_uuid=headers[key]["uuid"], + expected_date=date, + expected_values=list(data_dict.values()), + expected_module=logger_name, + expected_key=key, + ) + + +@pytest.mark.parametrize("level", LOGGING_LEVELS, ids=_logging.getLevelName) +@pytest.mark.parametrize( + "data_type,data", + list( + chain( + zip([str] * len(STRING_DATA_VALUES), STRING_DATA_VALUES), + product(SUPPORTED_ITERABLE_TYPES, ITERABLE_DATA_VALUES), + product(SUPPORTED_MAPPING_TYPES, MAPPING_DATA_VALUES), + ) + ), +) +@pytest.mark.parametrize("logger_name", LOGGER_NAMES) +@pytest.mark.parametrize("key", STRING_DATA_VALUES) +@pytest.mark.parametrize("description", [None, "test"]) +@pytest.mark.parametrize("number_repeats", [1, 2, 3]) +def test_logging_structured_data( + level: core.LogLevel, + data_type: Callable, + data: Mapping | Iterable, + logger_name: str, + key: str, + description: str, + number_repeats: int, + caplog: pytest.LogCaptureFixture, +) -> None: + logger = _setup_caplog_and_get_logger(caplog, logger_name, level) + log_data = data_type(data) + data_dict = logging.core._get_log_data_as_dict(log_data) + with _propagate_to_root(): + for _ in range(number_repeats): + logger.log(level=level, key=key, data=log_data, description=description) + assert len(caplog.records) == number_repeats + _parse_and_check_log_records( + caplog=caplog, + logger_name=logger_name, + logger_level=level, + message_level=level, + data_dicts=repeat(data_dict), + dates=repeat(SIMULATION_DATE), + keys=repeat(key), + description=description, + ) + + +@pytest.mark.parametrize("simulation_date_getter", [UpdateableSimulateDateGetter()]) +@pytest.mark.parametrize("logger_name", LOGGER_NAMES) +@pytest.mark.parametrize("number_dates", [2, 3]) +def test_logging_updating_simulation_date( + simulation_date_getter: core.SimulationDateGetter, + logger_name: str, + root_level: core.LogLevel, + number_dates: int, + caplog: pytest.LogCaptureFixture, +) -> None: + logger = _setup_caplog_and_get_logger(caplog, logger_name, root_level) + key = "message" + data = "spam" + data_dict = logging.core._get_log_data_as_dict(data) + dates = [] + with _propagate_to_root(): + for _ in range(number_dates): + logger.log(level=root_level, key=key, data=data) + dates.append(simulation_date_getter()) + simulation_date_getter.increment_date() + # Dates should be unique + assert len(set(dates)) == len(dates) + assert len(caplog.records) == number_dates + _parse_and_check_log_records( + caplog=caplog, + logger_name=logger_name, + logger_level=root_level, + message_level=root_level, + data_dicts=repeat(data_dict), + dates=dates, + keys=repeat(key), + description=None, + ) + + +@pytest.mark.parametrize("logger_name", LOGGER_NAMES) +def test_logging_structured_data_multiple_keys( + logger_name: str, + root_level: core.LogLevel, + caplog: pytest.LogCaptureFixture, +) -> None: + logger = _setup_caplog_and_get_logger(caplog, logger_name, root_level) + keys = ["foo", "bar", "foo", "foo", "bar"] + data_values = ["a", "b", "c", "d", "e"] + data_dicts = [logging.core._get_log_data_as_dict(data) for data in data_values] + with _propagate_to_root(): + for key, data in zip(keys, data_values): + logger.log(level=root_level, key=key, data=data) + assert len(caplog.records) == len(keys) + _parse_and_check_log_records( + caplog=caplog, + logger_name=logger_name, + logger_level=root_level, + message_level=root_level, + data_dicts=data_dicts, + dates=repeat(SIMULATION_DATE), + keys=keys, + description=None, + ) + + +@pytest.mark.parametrize("level", LOGGING_LEVELS) +def test_logging_to_file(level: core.LogLevel, tmp_path: Path) -> None: + log_path = tmp_path / "test.log" + file_handler = logging.set_output_file(log_path) + loggers = [logging.getLogger(name) for name in LOGGER_NAMES] + key = "message" + for logger, data in zip(loggers, STRING_DATA_VALUES): + logger.setLevel(level) + logger.log(level=level, key=key, data=data) + _logging.shutdown([lambda: file_handler]) + with log_path.open("r") as log_file: + log_lines = log_file.readlines() + # Should have two lines (one header + one data row per logger) + assert len(log_lines) == 2 * len(loggers) + for name, data in zip(LOGGER_NAMES, STRING_DATA_VALUES): + header = json.loads(log_lines.pop(0)) + row = json.loads(log_lines.pop(0)) + _check_header( + header=header, + expected_module=name, + expected_key=key, + expected_level=_logging.getLevelName(level), + expected_description=None, + expected_columns={key: "str"}, + ) + _check_row( + row=row, + logger_level=level, + expected_uuid=header["uuid"], + expected_date=SIMULATION_DATE, + expected_values=[data], + expected_module=name, + expected_key=key, + ) + + +@pytest.mark.parametrize( + "inconsistent_data_iterables", + [ + ({"a": 1, "b": 2}, {"a": 3, "b": 4, "c": 5}), + ({"a": 1}, {"b": 2}), + ({"a": None, "b": 2}, {"a": 1, "b": 2}), + ([1], [0.5]), + (["a", "b"], ["a", "b", "c"]), + ("foo", "bar", ["spam"]), + ], +) +def test_logging_structured_data_inconsistent_columns_warns( + inconsistent_data_iterables: Iterable[core.LogData], root_level: core.LogLevel +) -> None: + logger = logging.getLogger("tlo") + with pytest.warns(core.InconsistentLoggedColumnsWarning): + for data in inconsistent_data_iterables: + logger.log(level=root_level, key="message", data=data) + + +@pytest.mark.parametrize( + "consistent_data_iterables", + [ + ([np.int64(1)], [2], [np.int32(1)]), + ([{"a": np.bool_(False)}, {"a": False}]), + ((1.5, 2), (np.float64(0), np.int64(2))), + ], +) +@pytest.mark.filterwarnings("error") +def test_logging_structured_data_mixed_numpy_python_scalars( + consistent_data_iterables: Iterable[core.LogData], root_level: core.LogLevel +) -> None: + logger = logging.getLogger("tlo") + # Should run without any exceptions + for data in consistent_data_iterables: + logger.log(level=root_level, key="message", data=data) diff --git a/tests/test_logging_end_to_end.py b/tests/test_logging_end_to_end.py index 5f055c95ab..944c3021c4 100644 --- a/tests/test_logging_end_to_end.py +++ b/tests/test_logging_end_to_end.py @@ -16,13 +16,13 @@ def log_input(): log_string = "\n".join(( "col1_str;hello;world;lorem;ipsum;dolor;sit", "col2_int;1;3;5;7;8;10", - "col3_float;2;4;6;8;9;null", + "col3_float;2.1;4.1;6.1;8.1;9.1;0.1", "col4_cat;cat1;cat1;cat2;cat2;cat1;cat2", - "col5_set;set();{'one'};{None};{'three','four'};{'eight'};set()", - "col6_list;[];['two'];[None];[5, 6, 7];[];[]", + "col5_set;{'zero'};{'one'};{'two'};{'three'};{'four'};{'five'}", + "col6_list;[1, 3];[2, 4];[0, 3];[5, 6];[7, 8];[9, 10]", "col7_date;2020-06-19T00:22:58.586101;2020-06-20T00:23:58.586101;2020-06-21T00:24:58.586101;2020-06-22T00:25" - ":58.586101;2020-06-23T00:25:58.586101;null", - "col8_fixed_list;['one', 1];['two', 2];[None, None];['three', 3];['four', 4];['five', 5]" + ":58.586101;2020-06-23T00:25:58.586101;2020-06-21T00:24:58.586101", + "col8_fixed_list;['one', 1];['two', 2];['three', 3];['three', 3];['four', 4];['five', 5]" )) # read in, then transpose log_input = pd.read_csv(StringIO(log_string), sep=';').T @@ -63,8 +63,6 @@ def log_path(tmpdir_factory, log_input, class_scoped_seed): # a logger connected to that simulation logger = logging.getLogger('tlo.test') logger.setLevel(logging.INFO) - # Allowing logging of entire dataframe only for testing - logger._disable_dataframe_logging = False # log data as dicts for index, row in log_input.iterrows(): @@ -76,15 +74,9 @@ def log_path(tmpdir_factory, log_input, class_scoped_seed): logger.info(key='rows_as_individuals', data=log_input.loc[[index]]) sim.date = sim.date + pd.DateOffset(days=1) - # log data as multi-row dataframe - for _ in range(2): - logger.info(key='multi_row_df', data=log_input) - sim.date = sim.date + pd.DateOffset(days=1) - # log data as fixed length list for item in log_input.col8_fixed_list.values: - logger.info(key='a_fixed_length_list', - data=item) + logger.info(key='a_fixed_length_list', data=item) sim.date = sim.date + pd.DateOffset(days=1) # log data as variable length list @@ -137,26 +129,12 @@ def test_rows_as_individuals(self, test_log_df, log_input): log_output.col4_cat = log_output.col4_cat.astype('category') assert log_input.equals(log_output) - def test_log_entire_df(self, test_log_df, log_input): - # get table to compare - log_output = test_log_df['multi_row_df'].drop(['date'], axis=1) - - # within nested dicts/entire df, need manual setting of special types - log_output.col4_cat = log_output.col4_cat.astype('category') - log_input.col5_set = log_input.col5_set.apply(list) - log_output.col7_date = log_output.col7_date.astype('datetime64[ns]') - # deal with index matching by resetting index - log_output.reset_index(inplace=True, drop=True) - expected_output = pd.concat((log_input, log_input), ignore_index=True) - - assert expected_output.equals(log_output) - def test_fixed_length_list(self, test_log_df): log_df = test_log_df['a_fixed_length_list'].drop(['date'], axis=1) expected_output = pd.DataFrame( - {'item_1': ['one', 'two', None, 'three', 'four', 'five'], - 'item_2': [1, 2, None, 3, 4, 5]} + {'item_1': ['one', 'two', 'three', 'three', 'four', 'five'], + 'item_2': [1, 2, 3, 3, 4, 5]} ) assert expected_output.equals(log_df) From c9e65ca53083df6eae6bd97a125138802a10e941 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:34:53 +0100 Subject: [PATCH 12/19] Updates to HTM Swtiching Method to Fix Issue #1411 (#1413) * updates from paper analyses for HIV, TB and malaria * remove unused import statements * fix imports * update filepath for malaria resource file * remove test_hiv_tb_scenarios.py * updated test_healthsystem.py: test_manipulation_of_service_availability as small population size over 7 days will not schedule VMMC through HIV module. Remove HIV_Prevention_Circumcision in assert statement for HIV services delivered in one week * change ipt_coverage in TB logger as conflicts with existing parameter * updated ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx with updated NTP2019 TB data * remove test code * remove test code * delete tmp resourcefiles * malaria code use person_id consistently instead of individual_id * use individual_id for demography.do_death() * style change to avoid conflicts with master * style change to avoid conflicts with master * fix conflicts with master * fix conflicts with master * merge in master * check property hv_date_last_ART correctly set * Manually add PostnatalCare_Comprehensive to policy priorities * edit fix * add schisto high infection as conditional predictor for bladder cancer * fix conditional predictor for active TB - should check presence of CardioMetabolicDisorders * add 'ss' prefix to properties from schisto module referenced in bladder_cancer.py * edit praziquantel code in schisto.py to use value from CMST in place of donated * add parameter rr_depr_hiv for risk of depression with HIV infection * tidy up linear models in depression, include conditional predictors for hiv infection and add comments * move hv_inf into conditional predictor for depression in initial population * convert lm for incident cancer (site_confined) to model with conditional predictors. Include HIV as risk factor. * add parameter rr_site_confined_hiv to other_adult_cancers.py * update other_adult_cancers write-up to include HIV as risk factor * update Depression.docx to include HIV as risk factor for depression * edit HIV in depression to include only HIV cases not virally suppressed * update other_adult_cancers.py linear model to include HIV as risk factor only if not virally suppressed * edit: HIV remains as risk factor for depression independent of treatment status * include HIV as risk factor for low grade dysplasia (oesophageal cancer). Update ResourceFile_Oesophageal_Cancer.xlsx * update linear model for low grade dysplasia to include HIV as conditional risk factor * update OesophagealCancer.docx write-up to include HIV risk * add condition hiv diagnosed for increased risk of depression * remove hiv as risk factor for oesophageal cancer * remove parameter for hiv as risk factor for oesophageal cancer * update OesophagealCancer.docx to remove hiv as risk factor * update value for weighted risk of other_adult_cancers with unsuppressed HIV * add rr_hiv to linear model. update ResourceFile_cmd_condition_onset.xlsx with rr_hiv, leave value=1.0 if no effect of hiv * update resourcefiles for CMD include rr_hiv for all, no effect of majority of processes * refactoring: * put the helper function for switching scenario into same file a ScenarioSwitcher class * put tests for class and helper function together (next step will be to rename and mock-up extended functionality) * add diabetes as risk for active TB and relapse. add params to ResourceFile_TB.xlsx replace linearmodels dict which was accidentally removed in depression.py * add diabetes as risk factor for tb death * add diabetes as risk factor for PLHIV with active TB and on TB treatment * add diabetes as risk factor for PLHIV with active TB and on TB treatment * set up run to check calibration of deaths and disability * add predictor high-intensity S. haematobium infection to risk of bladder cancer in initial population * add predictor high-intensity S. haematobium infection to risk of HIV acquisition * fix indenting in HSI_Hiv_StartOrContinueTreatment * add hv_date_treated abd hv_date_last_ART to baseline_art * convert linear model in CMD to include conditional predictors * delete resourcefile created in error * comment out path-specific changes to analysis_cause_of_death_and_disability_calibrations.py * fix CMD error if Hiv not registered * initial sketch of structure * tidy-up and fix tests * commments * remove parameter rr_bcg_inf from tb.py * edit comment in initialise_simulation * fix parameter name error * update parameters * test runs * edit and fix flake8 errors * fix failing test * update ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx * update ResourceFile_PriorityRanking_ALLPOLICIES.xlsx * updated ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx * scenario file * starting work on the scenario file * design and HR scenarios * scenario for HRH * roll back changes to ScenarioSwitcher * Revert "roll back changes to ScenarioSwitcher" This reverts commit 49d14e767e0a98741478e08470e4043ec0908bd6. * sketch out of prposed changed for scenario_switcher * sketch out of prposed changed for scenario_switcher * sketch out of prposed changed for scenario_switcher * sketch out of prposed changed for scenario_switcher * sketch out of prposed changed for scenario_switcher * Add ResourceFile_Consumables_Item_Designations.csv * create special scenarios for consumables availability based on the designation of the consumable item * comment and fix imports * draft of scenario file * linting * increase length of scenarios and reduce reps * rename * correct error in specifiction of dynamic scaling scenario * remove trailing commas that casts return as tuple * Initially only consider baseline and perfect healthcare seeking scenarios, to get upper limit on RAM requirements * Submit remaining scenarios * direct to self.module (rather than self) * edit check for last ART when new dispensation occurs * update schisto risk on HIV to only include women * add scale-up parameter for htm programs add event to hiv module to switch parameters include new keywords in hiv module for scale-up separately for hiv, tb and malaria add new sheet in ResourceFile_HIV.xlsx containing new scale-up parameter values * add catch for malaria rdt_testing_rates post-2024 * malaria parameters scale-up included * restore `scenario_comparison_of_horizontal_and_vertical_programs` (removing comments used to select certain scenarios to be run). * first draft of figures * set up test for HTM scenario scale-up * add tests for HTM scale-up * check resourcefiles updated * check the usage of '' versus "" * reset to single quotes to match PR #1273 * remove unneeded resource files * remove unneeded resource files * edit filepaths * isort for imports * edit ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx to remove 2024 missing value in malaria Rate_rdt_testing * fix filename for test_HTMscaleup.py * update scenario file * update scenario file * include "scale_to_effective_capabilities": True * linting * linting * reduce number of draws * set scaleup parameters separately in each module use parameters to select scaleup instead of module arguments update resourcefiles * set scaleup parameters separately in each module use parameters to select scaleup instead of module arguments update resourcefiles * update resourcefiles * update resourcefiles * set up script to test scenarios - scale-up of HTM programs * test runs * add scenario switch to malaria.py * cherry-pick inadvertently updated files and revert * isort fixes * Delete resources/~$ResourceFile_HIV.xlsx * Delete resources/~$ResourceFile_TB.xlsx * Delete resources/malaria/~$ResourceFile_malaria.xlsx * fix failing tests * rollback changes to calibration_analyses/scenarios/long_run_all_diseases.py * fix error in filename * revert timedelta format to 'M' * add todos * merge in master * set up test runs for scale-up * edit scenario start date * change parameter scaleup_start_date to an integer value used in DateOffset, as timestamp is not json serializable for analysis runs * change scale-up event scheduling to use new DateOffset parameter * test runs * set up test runs * set up test runs * fix failing tests * Update tests/test_HTMscaleup.py Co-authored-by: Tim Hallett <39991060+tbhallett@users.noreply.github.com> * address comments on PR review * isort fixes * isort fixes * change np.timedelta in enhanced_lifestyle.py back to original * remove json file * call it 'htm_scenario_analysis' rather than just 'scenario_analysis' * update comment * roll back change in test_tb.py * remove .py extension for clarity * roll back incidental change * linting and editing string for clarity * roll back incidental changes * defaults for healthsystem ok -- no need to step through each option * remove inadvertent duplication in code * remove comment * use dict for ease of accessing * parameter to be the YEAR (int) of the change to fit with the convention used in other modules (instead of years since the beginning of the simulation) * remove comment * refactor module method for clarity * refactor to prevent same name being used for events specific to different modules * specify year for scale-up in analysis file * rename; add note; remove comments for HTM scenarios * refacotring * rename scenario class for hss elements * new scenario for combinations of vertical and horizontal programs * minor refactor * update docstring * renaming * add @property decorator * plot DALYS averted relative to Baseline - broken down by major cause (HIV, TB, MALARIA) * remove unused parameter prob_start_art_after_hiv_test * set up test runs * update linear models for tb and malaria after changing parameters * set up test runs * set up test runs * remove unused parameter from ResourceFile_HIV.xlsx * remove annual_testing_rate_children as not used * add scale-up parameter probability_of_being_retained_on_art_every_3_months, increase to 1 * set cons_availability to all and set off larger runs * test runs * change spec. of runs to be smaller pop size and fewer runs * roll back changes that are not related to this PR * delete file added by accident * roll back change (I think this is mistake) * refactor * remove clause `if p["do_scale_up"]` as not neccessary * make "_build_linear_models" function, called from initialise_population and update_parameter_for_program_scaleup --------- Co-authored-by: tdm32 Co-authored-by: Tara <37845078+tdm32@users.noreply.github.com> Co-authored-by: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> --- resources/ResourceFile_HIV.xlsx | 4 +- .../analysis_logged_deviance.py | 8 +- .../analysis_htm_scaleup.py | 8 +- .../htm_scenario_analyses/scenario_plots.py | 17 +++-- src/tlo/methods/hiv.py | 60 ++++++++------- src/tlo/methods/malaria.py | 75 ++++++++++--------- src/tlo/methods/tb.py | 42 ++++++----- 7 files changed, 112 insertions(+), 102 deletions(-) diff --git a/resources/ResourceFile_HIV.xlsx b/resources/ResourceFile_HIV.xlsx index f76169e701..b2db25c898 100644 --- a/resources/ResourceFile_HIV.xlsx +++ b/resources/ResourceFile_HIV.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:913d736db7717519270d61824a8855cbfd4d6e61a73b7ce51e2c3b7915b011ff -size 161597 +oid sha256:f57142efaf515d74f8290238ce1abad7b99871f9195623112892b3bb535bf634 +size 161721 diff --git a/src/scripts/hiv/projections_jan2023/analysis_logged_deviance.py b/src/scripts/hiv/projections_jan2023/analysis_logged_deviance.py index eca9f999bc..7a2af7fbed 100644 --- a/src/scripts/hiv/projections_jan2023/analysis_logged_deviance.py +++ b/src/scripts/hiv/projections_jan2023/analysis_logged_deviance.py @@ -34,8 +34,8 @@ # %% Run the simulation start_date = Date(2010, 1, 1) -end_date = Date(2014, 1, 1) -popsize = 1000 +end_date = Date(2022, 1, 1) +popsize = 5000 # scenario = 1 @@ -87,8 +87,8 @@ ) # set the scenario -# sim.modules["Hiv"].parameters["beta"] = 0.129671 -# sim.modules["Tb"].parameters["scaling_factor_WHO"] = 1.5 +sim.modules["Hiv"].parameters["do_scaleup"] = True +sim.modules["Hiv"].parameters["scaleup_start_year"] = 2019 # sim.modules["Tb"].parameters["scenario"] = scenario # sim.modules["Tb"].parameters["scenario_start_date"] = Date(2010, 1, 1) # sim.modules["Tb"].parameters["scenario_SI"] = "z" diff --git a/src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py b/src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py index a89231f670..beacb5e218 100644 --- a/src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py +++ b/src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py @@ -51,9 +51,9 @@ def __init__(self): super().__init__() self.seed = 0 self.start_date = Date(2010, 1, 1) - self.end_date = Date(2020, 1, 1) - self.pop_size = 75_000 - self.number_of_draws = 5 + self.end_date = Date(2025, 1, 1) + self.pop_size = 5_000 + self.number_of_draws = 2 self.runs_per_draw = 1 def log_configuration(self): @@ -86,7 +86,7 @@ def modules(self): ] def draw_parameters(self, draw_number, rng): - scaleup_start_year = 2012 + scaleup_start_year = 2019 return { 'Hiv': { diff --git a/src/scripts/htm_scenario_analyses/scenario_plots.py b/src/scripts/htm_scenario_analyses/scenario_plots.py index d14454ae13..c209c60f6e 100644 --- a/src/scripts/htm_scenario_analyses/scenario_plots.py +++ b/src/scripts/htm_scenario_analyses/scenario_plots.py @@ -23,6 +23,7 @@ datestamp = datetime.date.today().strftime("__%Y_%m_%d") outputspath = Path("./outputs") +# outputspath = Path("./outputs/t.mangal@imperial.ac.uk") # 0) Find results_folder associated with a given batch_file (and get most recent [-1]) @@ -32,7 +33,7 @@ make_graph_file_name = lambda stub: results_folder / f"{stub}.png" # noqa: E731 # look at one log (so can decide what to extract) -log = load_pickled_dataframes(results_folder) +log = load_pickled_dataframes(results_folder, draw=1) # get basic information about the results info = get_scenario_info(results_folder) @@ -55,14 +56,14 @@ def get_num_deaths_by_cause_label(_df): .size() -TARGET_PERIOD = (Date(2015, 1, 1), Date(2020, 1, 1)) +TARGET_PERIOD = (Date(2020, 1, 1), Date(2025, 1, 1)) num_deaths_by_cause_label = extract_results( results_folder, module='tlo.methods.demography', key='death', custom_generate_series=get_num_deaths_by_cause_label, - do_scaling=True + do_scaling=False ) @@ -103,9 +104,9 @@ def summarise_deaths_for_one_cause(results_folder, label): tb_deaths = summarise_deaths_for_one_cause(results_folder, 'TB (non-AIDS)') malaria_deaths = summarise_deaths_for_one_cause(results_folder, 'Malaria') -draw_labels = ['No scale-up', 'HIV, scale-up', 'TB scale-up', 'Malaria scale-up'] +draw_labels = ['No scale-up', 'HIV, scale-up', 'TB scale-up', 'Malaria scale-up', 'HTM scale-up'] -colors = sns.color_palette("Set1", 4) # Blue, Orange, Green, Red +colors = sns.color_palette("Set1", 5) # Blue, Orange, Green, Red # Create subplots @@ -116,19 +117,19 @@ def summarise_deaths_for_one_cause(results_folder, label): axs[0].plot(aids_deaths.index, aids_deaths[col], label=draw_labels[i], color=colors[i]) axs[0].set_title('HIV/AIDS') axs[0].legend() -axs[0].axvline(x=2015, color='gray', linestyle='--') +axs[0].axvline(x=2019, color='gray', linestyle='--') # Plot for df2 for i, col in enumerate(tb_deaths.columns): axs[1].plot(tb_deaths.index, tb_deaths[col], color=colors[i]) axs[1].set_title('TB') -axs[1].axvline(x=2015, color='gray', linestyle='--') +axs[1].axvline(x=2019, color='gray', linestyle='--') # Plot for df3 for i, col in enumerate(malaria_deaths.columns): axs[2].plot(malaria_deaths.index, malaria_deaths[col], color=colors[i]) axs[2].set_title('Malaria') -axs[2].axvline(x=2015, color='gray', linestyle='--') +axs[2].axvline(x=2019, color='gray', linestyle='--') for ax in axs: ax.set_xlabel('Years') diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py index 4c4e5d9c14..d744c9d254 100644 --- a/src/tlo/methods/hiv.py +++ b/src/tlo/methods/hiv.py @@ -472,10 +472,13 @@ def read_parameters(self, data_folder): ) def pre_initialise_population(self): - """ - * Establish the Linear Models - * - """ + """Do things required before the population is created + * Build the LinearModels""" + self._build_linear_models() + + def _build_linear_models(self): + """Establish the Linear Models""" + p = self.parameters # ---- LINEAR MODELS ----- @@ -1099,42 +1102,41 @@ def initialise_simulation(self, sim): ) def update_parameters_for_program_scaleup(self): - p = self.parameters scaled_params = p["scaleup_parameters"] - if p["do_scaleup"]: + # scale-up HIV program + # reduce risk of HIV - applies to whole adult population + p["beta"] = p["beta"] * scaled_params["reduction_in_hiv_beta"] - # scale-up HIV program - # reduce risk of HIV - applies to whole adult population - p["beta"] = p["beta"] * scaled_params["reduction_in_hiv_beta"] + # increase PrEP coverage for FSW after HIV test + p["prob_prep_for_fsw_after_hiv_test"] = scaled_params["prob_prep_for_fsw_after_hiv_test"] - # increase PrEP coverage for FSW after HIV test - p["prob_prep_for_fsw_after_hiv_test"] = scaled_params["prob_prep_for_fsw_after_hiv_test"] + # prep poll for AGYW - target to the highest risk + # increase retention to 75% for FSW and AGYW + p["prob_prep_for_agyw"] = scaled_params["prob_prep_for_agyw"] + p["probability_of_being_retained_on_prep_every_3_months"] = scaled_params["probability_of_being_retained_on_prep_every_3_months"] - # prep poll for AGYW - target to the highest risk - # increase retention to 75% for FSW and AGYW - p["prob_prep_for_agyw"] = scaled_params["prob_prep_for_agyw"] - p["probability_of_being_retained_on_prep_every_3_months"] = scaled_params["probability_of_being_retained_on_prep_every_3_months"] + # perfect retention on ART + p["probability_of_being_retained_on_art_every_3_months"] = scaled_params["probability_of_being_retained_on_art_every_3_months"] - # increase probability of VMMC after hiv test - p["prob_circ_after_hiv_test"] = scaled_params["prob_circ_after_hiv_test"] + # increase probability of VMMC after hiv test + p["prob_circ_after_hiv_test"] = scaled_params["prob_circ_after_hiv_test"] - # increase testing/diagnosis rates, default 2020 0.03/0.25 -> 93% dx - p["hiv_testing_rates"]["annual_testing_rate_children"] = scaled_params["annual_testing_rate_children"] - p["hiv_testing_rates"]["annual_testing_rate_adults"] = scaled_params["annual_testing_rate_adults"] + # increase testing/diagnosis rates, default 2020 0.03/0.25 -> 93% dx + p["hiv_testing_rates"]["annual_testing_rate_adults"] = scaled_params["annual_testing_rate_adults"] - # ANC testing - value for mothers and infants testing - p["prob_hiv_test_at_anc_or_delivery"] = scaled_params["prob_hiv_test_at_anc_or_delivery"] - p["prob_hiv_test_for_newborn_infant"] = scaled_params["prob_hiv_test_for_newborn_infant"] + # ANC testing - value for mothers and infants testing + p["prob_hiv_test_at_anc_or_delivery"] = scaled_params["prob_hiv_test_at_anc_or_delivery"] + p["prob_hiv_test_for_newborn_infant"] = scaled_params["prob_hiv_test_for_newborn_infant"] - # prob ART start if dx, this is already 95% at 2020 - p["prob_start_art_after_hiv_test"] = scaled_params["prob_start_art_after_hiv_test"] + # viral suppression rates + # adults already at 95% by 2020 + # change all column values + p["prob_start_art_or_vs"]["virally_suppressed_on_art"] = scaled_params["virally_suppressed_on_art"] - # viral suppression rates - # adults already at 95% by 2020 - # change all column values - p["prob_start_art_or_vs"]["virally_suppressed_on_art"] = scaled_params["virally_suppressed_on_art"] + # update exising linear models to use new scaled-up paramters + self._build_linear_models() def on_birth(self, mother_id, child_id): """ diff --git a/src/tlo/methods/malaria.py b/src/tlo/methods/malaria.py index 8c451b62d0..a5e9738a99 100644 --- a/src/tlo/methods/malaria.py +++ b/src/tlo/methods/malaria.py @@ -327,13 +327,16 @@ def read_parameters(self, data_folder): ) def pre_initialise_population(self): - """ - * Establish the Linear Models + """Do things required before the population is created + * Build the LinearModels""" + self._build_linear_models() + + def _build_linear_models(self): + """Establish the Linear Models if HIV is registered, the conditional predictors will apply otherwise only IPTp will affect risk of clinical/severe malaria """ - p = self.parameters # ---- LINEAR MODELS ----- @@ -656,52 +659,52 @@ def initialise_simulation(self, sim): ) def update_parameters_for_program_scaleup(self): - p = self.parameters scaled_params = p["scaleup_parameters"] - if p["do_scaleup"]: + # scale-up malaria program + # increase testing + # prob_malaria_case_tests=0.4 default + p["prob_malaria_case_tests"] = scaled_params["prob_malaria_case_tests"] - # scale-up malaria program - # increase testing - # prob_malaria_case_tests=0.4 default - p["prob_malaria_case_tests"] = scaled_params["prob_malaria_case_tests"] + # gen pop testing rates + # annual Rate_rdt_testing=0.64 at 2023 + p["rdt_testing_rates"]["Rate_rdt_testing"] = scaled_params["rdt_testing_rates"] - # gen pop testing rates - # annual Rate_rdt_testing=0.64 at 2023 - p["rdt_testing_rates"]["Rate_rdt_testing"] = scaled_params["rdt_testing_rates"] + # treatment reaches XX + # no default between testing and treatment, governed by tx availability - # treatment reaches XX - # no default between testing and treatment, governed by tx availability + # coverage IPTp reaches XX + # given during ANC visits and MalariaIPTp Event which selects ALL eligible women - # coverage IPTp reaches XX - # given during ANC visits and MalariaIPTp Event which selects ALL eligible women + # treatment success reaches 1 - default is currently 1 also + p["prob_of_treatment_success"] = scaled_params["prob_of_treatment_success"] - # treatment success reaches 1 - default is currently 1 also - p["prob_of_treatment_success"] = scaled_params["prob_of_treatment_success"] + # bednet and ITN coverage + # set IRS for 4 high-risk districts + # lookup table created in malaria read_parameters + # produces self.itn_irs called by malaria poll to draw incidence + # need to overwrite this + highrisk_distr_num = p["highrisk_districts"]["district_num"] - # bednet and ITN coverage - # set IRS for 4 high-risk districts - # lookup table created in malaria read_parameters - # produces self.itn_irs called by malaria poll to draw incidence - # need to overwrite this - highrisk_distr_num = p["highrisk_districts"]["district_num"] + # Find indices where District_Num is in highrisk_distr_num + mask = self.itn_irs['irs_rate'].index.get_level_values('District_Num').isin( + highrisk_distr_num) - # Find indices where District_Num is in highrisk_distr_num - mask = self.itn_irs['irs_rate'].index.get_level_values('District_Num').isin( - highrisk_distr_num) + # IRS values can be 0 or 0.8 - no other value in lookup table + self.itn_irs['irs_rate'].loc[mask] = scaled_params["irs_district"] - # IRS values can be 0 or 0.8 - no other value in lookup table - self.itn_irs['irs_rate'].loc[mask] = scaled_params["irs_district"] + # set ITN for all districts + # Set these values to 0.7 - this is the max value possible in lookup table + # equivalent to 0.7 of all pop sleeping under bednet + # household coverage could be 100%, but not everyone in household sleeping under bednet + self.itn_irs['itn_rate'] = scaled_params["itn_district"] - # set ITN for all districts - # Set these values to 0.7 - this is the max value possible in lookup table - # equivalent to 0.7 of all pop sleeping under bednet - # household coverage could be 100%, but not everyone in household sleeping under bednet - self.itn_irs['itn_rate'] = scaled_params["itn_district"] + # itn rates for 2019 onwards + p["itn"] = scaled_params["itn"] - # itn rates for 2019 onwards - p["itn"] = scaled_params["itn"] + # update exising linear models to use new scaled-up parameters + self._build_linear_models() def on_birth(self, mother_id, child_id): df = self.sim.population.props diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py index 5f87bcd261..3392d90b62 100644 --- a/src/tlo/methods/tb.py +++ b/src/tlo/methods/tb.py @@ -470,9 +470,13 @@ def read_parameters(self, data_folder): ) def pre_initialise_population(self): - """ - * Establish the Linear Models - """ + """Do things required before the population is created + * Build the LinearModels""" + self._build_linear_models() + + def _build_linear_models(self): + """Establish the Linear Models""" + p = self.parameters # risk of active tb @@ -885,29 +889,29 @@ def initialise_simulation(self, sim): ) def update_parameters_for_program_scaleup(self): - p = self.parameters scaled_params = p["scaleup_parameters"] - if p["do_scaleup"]: + # scale-up TB program + # use NTP treatment rates + p["rate_testing_active_tb"]["treatment_coverage"] = scaled_params["tb_treatment_coverage"] - # scale-up TB program - # use NTP treatment rates - p["rate_testing_active_tb"]["treatment_coverage"] = scaled_params["tb_treatment_coverage"] + # increase tb treatment success rates + p["prob_tx_success_ds"] = scaled_params["tb_prob_tx_success_ds"] + p["prob_tx_success_mdr"] = scaled_params["tb_prob_tx_success_mdr"] + p["prob_tx_success_0_4"] = scaled_params["tb_prob_tx_success_0_4"] + p["prob_tx_success_5_14"] = scaled_params["tb_prob_tx_success_5_14"] - # increase tb treatment success rates - p["prob_tx_success_ds"] = scaled_params["tb_prob_tx_success_ds"] - p["prob_tx_success_mdr"] = scaled_params["tb_prob_tx_success_mdr"] - p["prob_tx_success_0_4"] = scaled_params["tb_prob_tx_success_0_4"] - p["prob_tx_success_5_14"] = scaled_params["tb_prob_tx_success_5_14"] + # change first-line testing for TB to xpert + p["first_line_test"] = scaled_params["first_line_test"] + p["second_line_test"] = scaled_params["second_line_test"] - # change first-line testing for TB to xpert - p["first_line_test"] = scaled_params["first_line_test"] - p["second_line_test"] = scaled_params["second_line_test"] + # increase coverage of IPT + p["ipt_coverage"]["coverage_plhiv"] = scaled_params["ipt_coverage_plhiv"] + p["ipt_coverage"]["coverage_paediatric"] = scaled_params["ipt_coverage_paediatric"] - # increase coverage of IPT - p["ipt_coverage"]["coverage_plhiv"] = scaled_params["ipt_coverage_plhiv"] - p["ipt_coverage"]["coverage_paediatric"] = scaled_params["ipt_coverage_paediatric"] + # update exising linear models to use new scaled-up paramters + self._build_linear_models() def on_birth(self, mother_id, child_id): """Initialise properties for a newborn individual From 068ee0c174d08057ff214d9e176e7411ce5fd846 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:47:01 +0100 Subject: [PATCH 13/19] Initial Version of Scenario definition and other associated amendments needed for the "Horizontal vs Vertical"-type Analysis, (#1288) * updates from paper analyses for HIV, TB and malaria * remove unused import statements * fix imports * update filepath for malaria resource file * remove test_hiv_tb_scenarios.py * updated test_healthsystem.py: test_manipulation_of_service_availability as small population size over 7 days will not schedule VMMC through HIV module. Remove HIV_Prevention_Circumcision in assert statement for HIV services delivered in one week * change ipt_coverage in TB logger as conflicts with existing parameter * updated ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx with updated NTP2019 TB data * remove test code * remove test code * delete tmp resourcefiles * malaria code use person_id consistently instead of individual_id * use individual_id for demography.do_death() * style change to avoid conflicts with master * style change to avoid conflicts with master * fix conflicts with master * fix conflicts with master * merge in master * check property hv_date_last_ART correctly set * Manually add PostnatalCare_Comprehensive to policy priorities * edit fix * add schisto high infection as conditional predictor for bladder cancer * fix conditional predictor for active TB - should check presence of CardioMetabolicDisorders * add 'ss' prefix to properties from schisto module referenced in bladder_cancer.py * edit praziquantel code in schisto.py to use value from CMST in place of donated * add parameter rr_depr_hiv for risk of depression with HIV infection * tidy up linear models in depression, include conditional predictors for hiv infection and add comments * move hv_inf into conditional predictor for depression in initial population * convert lm for incident cancer (site_confined) to model with conditional predictors. Include HIV as risk factor. * add parameter rr_site_confined_hiv to other_adult_cancers.py * update other_adult_cancers write-up to include HIV as risk factor * update Depression.docx to include HIV as risk factor for depression * edit HIV in depression to include only HIV cases not virally suppressed * update other_adult_cancers.py linear model to include HIV as risk factor only if not virally suppressed * edit: HIV remains as risk factor for depression independent of treatment status * include HIV as risk factor for low grade dysplasia (oesophageal cancer). Update ResourceFile_Oesophageal_Cancer.xlsx * update linear model for low grade dysplasia to include HIV as conditional risk factor * update OesophagealCancer.docx write-up to include HIV risk * add condition hiv diagnosed for increased risk of depression * remove hiv as risk factor for oesophageal cancer * remove parameter for hiv as risk factor for oesophageal cancer * update OesophagealCancer.docx to remove hiv as risk factor * update value for weighted risk of other_adult_cancers with unsuppressed HIV * add rr_hiv to linear model. update ResourceFile_cmd_condition_onset.xlsx with rr_hiv, leave value=1.0 if no effect of hiv * update resourcefiles for CMD include rr_hiv for all, no effect of majority of processes * refactoring: * put the helper function for switching scenario into same file a ScenarioSwitcher class * put tests for class and helper function together (next step will be to rename and mock-up extended functionality) * add diabetes as risk for active TB and relapse. add params to ResourceFile_TB.xlsx replace linearmodels dict which was accidentally removed in depression.py * add diabetes as risk factor for tb death * add diabetes as risk factor for PLHIV with active TB and on TB treatment * add diabetes as risk factor for PLHIV with active TB and on TB treatment * set up run to check calibration of deaths and disability * add predictor high-intensity S. haematobium infection to risk of bladder cancer in initial population * add predictor high-intensity S. haematobium infection to risk of HIV acquisition * fix indenting in HSI_Hiv_StartOrContinueTreatment * add hv_date_treated abd hv_date_last_ART to baseline_art * convert linear model in CMD to include conditional predictors * delete resourcefile created in error * comment out path-specific changes to analysis_cause_of_death_and_disability_calibrations.py * fix CMD error if Hiv not registered * initial sketch of structure * tidy-up and fix tests * commments * remove parameter rr_bcg_inf from tb.py * edit comment in initialise_simulation * fix parameter name error * update parameters * test runs * edit and fix flake8 errors * fix failing test * update ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx * update ResourceFile_PriorityRanking_ALLPOLICIES.xlsx * updated ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx * scenario file * starting work on the scenario file * design and HR scenarios * scenario for HRH * roll back changes to ScenarioSwitcher * Revert "roll back changes to ScenarioSwitcher" This reverts commit 49d14e767e0a98741478e08470e4043ec0908bd6. * sketch out of prposed changed for scenario_switcher * sketch out of prposed changed for scenario_switcher * sketch out of prposed changed for scenario_switcher * sketch out of prposed changed for scenario_switcher * sketch out of prposed changed for scenario_switcher * Add ResourceFile_Consumables_Item_Designations.csv * create special scenarios for consumables availability based on the designation of the consumable item * comment and fix imports * draft of scenario file * linting * increase length of scenarios and reduce reps * rename * correct error in specifiction of dynamic scaling scenario * remove trailing commas that casts return as tuple * Initially only consider baseline and perfect healthcare seeking scenarios, to get upper limit on RAM requirements * Submit remaining scenarios * direct to self.module (rather than self) * edit check for last ART when new dispensation occurs * update schisto risk on HIV to only include women * add scale-up parameter for htm programs add event to hiv module to switch parameters include new keywords in hiv module for scale-up separately for hiv, tb and malaria add new sheet in ResourceFile_HIV.xlsx containing new scale-up parameter values * add catch for malaria rdt_testing_rates post-2024 * malaria parameters scale-up included * restore `scenario_comparison_of_horizontal_and_vertical_programs` (removing comments used to select certain scenarios to be run). * first draft of figures * set up test for HTM scenario scale-up * add tests for HTM scale-up * check resourcefiles updated * check the usage of '' versus "" * reset to single quotes to match PR #1273 * remove unneeded resource files * remove unneeded resource files * edit filepaths * isort for imports * edit ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx to remove 2024 missing value in malaria Rate_rdt_testing * fix filename for test_HTMscaleup.py * update scenario file * update scenario file * include "scale_to_effective_capabilities": True * linting * linting * reduce number of draws * set scaleup parameters separately in each module use parameters to select scaleup instead of module arguments update resourcefiles * set scaleup parameters separately in each module use parameters to select scaleup instead of module arguments update resourcefiles * update resourcefiles * update resourcefiles * set up script to test scenarios - scale-up of HTM programs * test runs * add scenario switch to malaria.py * cherry-pick inadvertently updated files and revert * isort fixes * Delete resources/~$ResourceFile_HIV.xlsx * Delete resources/~$ResourceFile_TB.xlsx * Delete resources/malaria/~$ResourceFile_malaria.xlsx * fix failing tests * rollback changes to calibration_analyses/scenarios/long_run_all_diseases.py * fix error in filename * revert timedelta format to 'M' * add todos * merge in master * set up test runs for scale-up * edit scenario start date * change parameter scaleup_start_date to an integer value used in DateOffset, as timestamp is not json serializable for analysis runs * change scale-up event scheduling to use new DateOffset parameter * test runs * set up test runs * set up test runs * fix failing tests * Update tests/test_HTMscaleup.py Co-authored-by: Tim Hallett <39991060+tbhallett@users.noreply.github.com> * address comments on PR review * isort fixes * isort fixes * change np.timedelta in enhanced_lifestyle.py back to original * remove json file * call it 'htm_scenario_analysis' rather than just 'scenario_analysis' * update comment * roll back change in test_tb.py * remove .py extension for clarity * roll back incidental change * linting and editing string for clarity * roll back incidental changes * defaults for healthsystem ok -- no need to step through each option * remove inadvertent duplication in code * remove comment * use dict for ease of accessing * parameter to be the YEAR (int) of the change to fit with the convention used in other modules (instead of years since the beginning of the simulation) * remove comment * refactor module method for clarity * refactor to prevent same name being used for events specific to different modules * specify year for scale-up in analysis file * rename; add note; remove comments for HTM scenarios * refacotring * rename scenario class for hss elements * new scenario for combinations of vertical and horizontal programs * minor refactor * update docstring * renaming * add @property decorator * plot DALYS averted relative to Baseline - broken down by major cause (HIV, TB, MALARIA) * define mini run scenario * rename scenario * lining isort * linting ruff * delete not-yet-used file * isort again! --------- Co-authored-by: tdm32 Co-authored-by: Tara <37845078+tdm32@users.noreply.github.com> Co-authored-by: Margherita Molaro <48129834+marghe-molaro@users.noreply.github.com> --- ..._HR_scaling_by_level_and_officer_type.xlsx | 4 +- .../analysis_hss_elements.py | 272 +++++++++++++ ..._vertical_programs_with_and_without_hss.py | 363 ++++++++++++++++++ .../mini_version_scenario.py | 109 ++++++ .../scenario_definitions.py | 90 +++++ .../scenario_hss_elements.py | 242 ++++++++++++ ..._vertical_programs_with_and_without_hss.py | 144 +++++++ 7 files changed, 1222 insertions(+), 2 deletions(-) create mode 100644 src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_hss_elements.py create mode 100644 src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_vertical_programs_with_and_without_hss.py create mode 100644 src/scripts/comparison_of_horizontal_and_vertical_programs/mini_analysis_for_testing/mini_version_scenario.py create mode 100644 src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py create mode 100644 src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_hss_elements.py create mode 100644 src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_vertical_programs_with_and_without_hss.py diff --git a/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_HR_scaling_by_level_and_officer_type.xlsx b/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_HR_scaling_by_level_and_officer_type.xlsx index 3d804bbc77..1dc4b6ea7e 100644 --- a/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_HR_scaling_by_level_and_officer_type.xlsx +++ b/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_HR_scaling_by_level_and_officer_type.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:af86c2c2af5c291c18c5d481681d6d316526b81806c8c8e898517e850160e6fd -size 12465 +oid sha256:80651d157772a292bf9617c86e2616d8165a20385ada6d85a5244aca9c55aa0c +size 21938 diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_hss_elements.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_hss_elements.py new file mode 100644 index 0000000000..76708f7c25 --- /dev/null +++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_hss_elements.py @@ -0,0 +1,272 @@ +"""Produce plots to show the impact each the healthcare system (overall health impact) when running under different +scenarios (scenario_impact_of_healthsystem.py)""" + +import argparse +import textwrap +from pathlib import Path +from typing import Tuple + +import numpy as np +import pandas as pd +from matplotlib import pyplot as plt + +from tlo import Date +from tlo.analysis.utils import extract_results, make_age_grp_lookup, summarize + + +def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = None): + """Produce standard set of plots describing the effect of each TREATMENT_ID. + - We estimate the epidemiological impact as the EXTRA deaths that would occur if that treatment did not occur. + - We estimate the draw on healthcare system resources as the FEWER appointments when that treatment does not occur. + """ + + TARGET_PERIOD = (Date(2020, 1, 1), Date(2030, 12, 31)) + + # Definitions of general helper functions + make_graph_file_name = lambda stub: output_folder / f"{stub.replace('*', '_star_')}.png" # noqa: E731 + + _, age_grp_lookup = make_age_grp_lookup() + + def target_period() -> str: + """Returns the target period as a string of the form YYYY-YYYY""" + return "-".join(str(t.year) for t in TARGET_PERIOD) + + def get_parameter_names_from_scenario_file() -> Tuple[str]: + """Get the tuple of names of the scenarios from `Scenario` class used to create the results.""" + from scripts.comparison_of_horizontal_and_vertical_programs.scenario_hss_elements import ( + HSSElements, + ) + e = HSSElements() + return tuple(e._scenarios.keys()) + + def get_num_deaths(_df): + """Return total number of Deaths (total within the TARGET_PERIOD)""" + return pd.Series(data=len(_df.loc[pd.to_datetime(_df.date).between(*TARGET_PERIOD)])) + + def get_num_dalys(_df): + """Return total number of DALYS (Stacked) by label (total within the TARGET_PERIOD). + Throw error if not a record for every year in the TARGET PERIOD (to guard against inadvertently using + results from runs that crashed mid-way through the simulation. + """ + years_needed = [i.year for i in TARGET_PERIOD] + assert set(_df.year.unique()).issuperset(years_needed), "Some years are not recorded." + return pd.Series( + data=_df + .loc[_df.year.between(*years_needed)] + .drop(columns=['date', 'sex', 'age_range', 'year']) + .sum().sum() + ) + + def set_param_names_as_column_index_level_0(_df): + """Set the columns index (level 0) as the param_names.""" + ordered_param_names_no_prefix = {i: x for i, x in enumerate(param_names)} + names_of_cols_level0 = [ordered_param_names_no_prefix.get(col) for col in _df.columns.levels[0]] + assert len(names_of_cols_level0) == len(_df.columns.levels[0]) + _df.columns = _df.columns.set_levels(names_of_cols_level0, level=0) + return _df + + def find_difference_relative_to_comparison(_ser: pd.Series, + comparison: str, + scaled: bool = False, + drop_comparison: bool = True, + ): + """Find the difference in the values in a pd.Series with a multi-index, between the draws (level 0) + within the runs (level 1), relative to where draw = `comparison`. + The comparison is `X - COMPARISON`.""" + return _ser \ + .unstack(level=0) \ + .apply(lambda x: (x - x[comparison]) / (x[comparison] if scaled else 1.0), axis=1) \ + .drop(columns=([comparison] if drop_comparison else [])) \ + .stack() + + def do_bar_plot_with_ci(_df, annotations=None, xticklabels_horizontal_and_wrapped=False, put_labels_in_legend=True): + """Make a vertical bar plot for each row of _df, using the columns to identify the height of the bar and the + extent of the error bar.""" + + substitute_labels = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + + yerr = np.array([ + (_df['mean'] - _df['lower']).values, + (_df['upper'] - _df['mean']).values, + ]) + + xticks = {(i + 0.5): k for i, k in enumerate(_df.index)} + + # Define colormap (used only with option `put_labels_in_legend=True`) + cmap = plt.get_cmap("tab20") + rescale = lambda y: (y - np.min(y)) / (np.max(y) - np.min(y)) # noqa: E731 + colors = list(map(cmap, rescale(np.array(list(xticks.keys()))))) if put_labels_in_legend else None + + fig, ax = plt.subplots(figsize=(10, 5)) + ax.bar( + xticks.keys(), + _df['mean'].values, + yerr=yerr, + alpha=0.8, + ecolor='black', + color=colors, + capsize=10, + label=xticks.values() + ) + if annotations: + for xpos, ypos, text in zip(xticks.keys(), _df['upper'].values, annotations): + ax.text(xpos, ypos*1.15, text, horizontalalignment='center', rotation='vertical', fontsize='x-small') + ax.set_xticks(list(xticks.keys())) + + if put_labels_in_legend: + # Update xticks label with substitute labels + # Insert legend with updated labels that shows correspondence between substitute label and original label + xtick_values = [letter for letter, label in zip(substitute_labels, xticks.values())] + xtick_legend = [f'{letter}: {label}' for letter, label in zip(substitute_labels, xticks.values())] + h, legs = ax.get_legend_handles_labels() + ax.legend(h, xtick_legend, loc='center left', fontsize='small', bbox_to_anchor=(1, 0.5)) + ax.set_xticklabels(list(xtick_values)) + else: + if not xticklabels_horizontal_and_wrapped: + # xticklabels will be vertical and not wrapped + ax.set_xticklabels(list(xticks.values()), rotation=90) + else: + wrapped_labs = ["\n".join(textwrap.wrap(_lab, 20)) for _lab in xticks.values()] + ax.set_xticklabels(wrapped_labs) + + ax.grid(axis="y") + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) + fig.tight_layout() + + return fig, ax + + # %% Define parameter names + param_names = get_parameter_names_from_scenario_file() + + # %% Quantify the health gains associated with all interventions combined. + + # Absolute Number of Deaths and DALYs + num_deaths = extract_results( + results_folder, + module='tlo.methods.demography', + key='death', + custom_generate_series=get_num_deaths, + do_scaling=True + ).pipe(set_param_names_as_column_index_level_0) + + num_dalys = extract_results( + results_folder, + module='tlo.methods.healthburden', + key='dalys_stacked', + custom_generate_series=get_num_dalys, + do_scaling=True + ).pipe(set_param_names_as_column_index_level_0) + + # %% Charts of total numbers of deaths / DALYS + num_dalys_summarized = summarize(num_dalys).loc[0].unstack().reindex(param_names) + num_deaths_summarized = summarize(num_deaths).loc[0].unstack().reindex(param_names) + + name_of_plot = f'Deaths, {target_period()}' + fig, ax = do_bar_plot_with_ci(num_deaths_summarized / 1e6) + ax.set_title(name_of_plot) + ax.set_ylabel('(Millions)') + fig.tight_layout() + ax.axhline(num_deaths_summarized.loc['Baseline', 'mean']/1e6, color='black', alpha=0.5) + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + name_of_plot = f'All Scenarios: DALYs, {target_period()}' + fig, ax = do_bar_plot_with_ci(num_dalys_summarized / 1e6) + ax.set_title(name_of_plot) + ax.set_ylabel('(Millions)') + ax.axhline(num_dalys_summarized.loc['Baseline', 'mean']/1e6, color='black', alpha=0.5) + fig.tight_layout() + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + + # %% Deaths and DALYS averted relative to Status Quo + num_deaths_averted = summarize( + -1.0 * + pd.DataFrame( + find_difference_relative_to_comparison( + num_deaths.loc[0], + comparison='Baseline') + ).T + ).iloc[0].unstack().reindex(param_names).drop(['Baseline']) + + pc_deaths_averted = 100.0 * summarize( + -1.0 * + pd.DataFrame( + find_difference_relative_to_comparison( + num_deaths.loc[0], + comparison='Baseline', + scaled=True) + ).T + ).iloc[0].unstack().reindex(param_names).drop(['Baseline']) + + num_dalys_averted = summarize( + -1.0 * + pd.DataFrame( + find_difference_relative_to_comparison( + num_dalys.loc[0], + comparison='Baseline') + ).T + ).iloc[0].unstack().reindex(param_names).drop(['Baseline']) + + pc_dalys_averted = 100.0 * summarize( + -1.0 * + pd.DataFrame( + find_difference_relative_to_comparison( + num_dalys.loc[0], + comparison='Baseline', + scaled=True) + ).T + ).iloc[0].unstack().reindex(param_names).drop(['Baseline']) + + # DEATHS + name_of_plot = f'Additional Deaths Averted vs Baseline, {target_period()}' + fig, ax = do_bar_plot_with_ci( + num_deaths_averted.clip(lower=0.0), + annotations=[ + f"{round(row['mean'], 0)} ({round(row['lower'], 1)}-{round(row['upper'], 1)}) %" + for _, row in pc_deaths_averted.clip(lower=0.0).iterrows() + ] + ) + ax.set_title(name_of_plot) + ax.set_ylabel('Additional Deaths Averted') + fig.tight_layout() + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + # DALYS + name_of_plot = f'Additional DALYs Averted vs Baseline, {target_period()}' + fig, ax = do_bar_plot_with_ci( + (num_dalys_averted / 1e6).clip(lower=0.0), + annotations=[ + f"{round(row['mean'])} ({round(row['lower'], 1)}-{round(row['upper'], 1)}) %" + for _, row in pc_dalys_averted.clip(lower=0.0).iterrows() + ] + ) + ax.set_title(name_of_plot) + ax.set_ylabel('Additional DALYS Averted \n(Millions)') + fig.tight_layout() + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + # todo: Neaten graphs + # todo: Graph showing difference broken down by disease (this can be cribbed from the calcs about wealth from the + # third set of analyses in the overview paper). + # todo: other metrics of health + # todo: other graphs, broken down by age/sex (this can also be cribbed from overview paper stuff) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("results_folder", type=Path) # outputs/horizontal_and_vertical_programs-2024-05-16 + args = parser.parse_args() + + apply( + results_folder=args.results_folder, + output_folder=args.results_folder, + resourcefilepath=Path('./resources') + ) diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_vertical_programs_with_and_without_hss.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_vertical_programs_with_and_without_hss.py new file mode 100644 index 0000000000..f0dd083d97 --- /dev/null +++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_vertical_programs_with_and_without_hss.py @@ -0,0 +1,363 @@ +"""Produce plots to show the impact each the healthcare system (overall health impact) when running under different +scenarios (scenario_impact_of_healthsystem.py)""" + +import argparse +import textwrap +from pathlib import Path +from typing import Tuple + +import numpy as np +import pandas as pd +from matplotlib import pyplot as plt + +from tlo import Date +from tlo.analysis.utils import extract_results, make_age_grp_lookup, summarize + + +def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = None): + """Produce standard set of plots describing the effect of each TREATMENT_ID. + - We estimate the epidemiological impact as the EXTRA deaths that would occur if that treatment did not occur. + - We estimate the draw on healthcare system resources as the FEWER appointments when that treatment does not occur. + """ + + TARGET_PERIOD = (Date(2020, 1, 1), Date(2030, 12, 31)) + + # Definitions of general helper functions + make_graph_file_name = lambda stub: output_folder / f"{stub.replace('*', '_star_')}.png" # noqa: E731 + + _, age_grp_lookup = make_age_grp_lookup() + + def target_period() -> str: + """Returns the target period as a string of the form YYYY-YYYY""" + return "-".join(str(t.year) for t in TARGET_PERIOD) + + def get_parameter_names_from_scenario_file() -> Tuple[str]: + """Get the tuple of names of the scenarios from `Scenario` class used to create the results.""" + from scripts.comparison_of_horizontal_and_vertical_programs.scenario_vertical_programs_with_and_without_hss import ( + HTMWithAndWithoutHSS, + ) + e = HTMWithAndWithoutHSS() + return tuple(e._scenarios.keys()) + + def get_num_deaths(_df): + """Return total number of Deaths (total within the TARGET_PERIOD)""" + return pd.Series(data=len(_df.loc[pd.to_datetime(_df.date).between(*TARGET_PERIOD)])) + + def get_num_dalys(_df): + """Return total number of DALYS (Stacked) by label (total within the TARGET_PERIOD). + Throw error if not a record for every year in the TARGET PERIOD (to guard against inadvertently using + results from runs that crashed mid-way through the simulation. + """ + years_needed = [i.year for i in TARGET_PERIOD] + assert set(_df.year.unique()).issuperset(years_needed), "Some years are not recorded." + return pd.Series( + data=_df + .loc[_df.year.between(*years_needed)] + .drop(columns=['date', 'sex', 'age_range', 'year']) + .sum().sum() + ) + + def set_param_names_as_column_index_level_0(_df): + """Set the columns index (level 0) as the param_names.""" + ordered_param_names_no_prefix = {i: x for i, x in enumerate(param_names)} + names_of_cols_level0 = [ordered_param_names_no_prefix.get(col) for col in _df.columns.levels[0]] + assert len(names_of_cols_level0) == len(_df.columns.levels[0]) + _df.columns = _df.columns.set_levels(names_of_cols_level0, level=0) + return _df + + def find_difference_relative_to_comparison_series( + _ser: pd.Series, + comparison: str, + scaled: bool = False, + drop_comparison: bool = True, + ): + """Find the difference in the values in a pd.Series with a multi-index, between the draws (level 0) + within the runs (level 1), relative to where draw = `comparison`. + The comparison is `X - COMPARISON`.""" + return _ser \ + .unstack(level=0) \ + .apply(lambda x: (x - x[comparison]) / (x[comparison] if scaled else 1.0), axis=1) \ + .drop(columns=([comparison] if drop_comparison else [])) \ + .stack() + + def find_difference_relative_to_comparison_series_dataframe(_df: pd.DataFrame, **kwargs): + """Apply `find_difference_relative_to_comparison_series` to each row in a dataframe""" + return pd.concat({ + _idx: find_difference_relative_to_comparison_series(row, **kwargs) + for _idx, row in _df.iterrows() + }, axis=1).T + + def do_bar_plot_with_ci(_df, annotations=None, xticklabels_horizontal_and_wrapped=False, put_labels_in_legend=True): + """Make a vertical bar plot for each row of _df, using the columns to identify the height of the bar and the + extent of the error bar.""" + + substitute_labels = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + + yerr = np.array([ + (_df['mean'] - _df['lower']).values, + (_df['upper'] - _df['mean']).values, + ]) + + xticks = {(i + 0.5): k for i, k in enumerate(_df.index)} + + # Define colormap (used only with option `put_labels_in_legend=True`) + cmap = plt.get_cmap("tab20") + rescale = lambda y: (y - np.min(y)) / (np.max(y) - np.min(y)) # noqa: E731 + colors = list(map(cmap, rescale(np.array(list(xticks.keys()))))) if put_labels_in_legend else None + + fig, ax = plt.subplots(figsize=(10, 5)) + ax.bar( + xticks.keys(), + _df['mean'].values, + yerr=yerr, + alpha=0.8, + ecolor='black', + color=colors, + capsize=10, + label=xticks.values() + ) + if annotations: + for xpos, ypos, text in zip(xticks.keys(), _df['upper'].values, annotations): + ax.text(xpos, ypos*1.15, text, horizontalalignment='center', rotation='vertical', fontsize='x-small') + ax.set_xticks(list(xticks.keys())) + + if put_labels_in_legend: + # Update xticks label with substitute labels + # Insert legend with updated labels that shows correspondence between substitute label and original label + xtick_values = [letter for letter, label in zip(substitute_labels, xticks.values())] + xtick_legend = [f'{letter}: {label}' for letter, label in zip(substitute_labels, xticks.values())] + h, legs = ax.get_legend_handles_labels() + ax.legend(h, xtick_legend, loc='center left', fontsize='small', bbox_to_anchor=(1, 0.5)) + ax.set_xticklabels(list(xtick_values)) + else: + if not xticklabels_horizontal_and_wrapped: + # xticklabels will be vertical and not wrapped + ax.set_xticklabels(list(xticks.values()), rotation=90) + else: + wrapped_labs = ["\n".join(textwrap.wrap(_lab, 20)) for _lab in xticks.values()] + ax.set_xticklabels(wrapped_labs) + + ax.grid(axis="y") + ax.spines['top'].set_visible(False) + ax.spines['right'].set_visible(False) + fig.tight_layout() + + return fig, ax + + # %% Define parameter names + param_names = get_parameter_names_from_scenario_file() + + # %% Quantify the health gains associated with all interventions combined. + + # Absolute Number of Deaths and DALYs + num_deaths = extract_results( + results_folder, + module='tlo.methods.demography', + key='death', + custom_generate_series=get_num_deaths, + do_scaling=True + ).pipe(set_param_names_as_column_index_level_0) + + num_dalys = extract_results( + results_folder, + module='tlo.methods.healthburden', + key='dalys_stacked', + custom_generate_series=get_num_dalys, + do_scaling=True + ).pipe(set_param_names_as_column_index_level_0) + + # %% Charts of total numbers of deaths / DALYS + num_dalys_summarized = summarize(num_dalys).loc[0].unstack().reindex(param_names) + num_deaths_summarized = summarize(num_deaths).loc[0].unstack().reindex(param_names) + + name_of_plot = f'Deaths, {target_period()}' + fig, ax = do_bar_plot_with_ci(num_deaths_summarized / 1e6) + ax.set_title(name_of_plot) + ax.set_ylabel('(Millions)') + fig.tight_layout() + ax.axhline(num_deaths_summarized.loc['Baseline', 'mean']/1e6, color='black', alpha=0.5) + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + name_of_plot = f'All Scenarios: DALYs, {target_period()}' + fig, ax = do_bar_plot_with_ci(num_dalys_summarized / 1e6) + ax.set_title(name_of_plot) + ax.set_ylabel('(Millions)') + ax.axhline(num_dalys_summarized.loc['Baseline', 'mean']/1e6, color='black', alpha=0.5) + fig.tight_layout() + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + + # %% Deaths and DALYS averted relative to Status Quo + num_deaths_averted = summarize( + -1.0 * + pd.DataFrame( + find_difference_relative_to_comparison_series( + num_deaths.loc[0], + comparison='Baseline') + ).T + ).iloc[0].unstack().reindex(param_names).drop(['Baseline']) + + pc_deaths_averted = 100.0 * summarize( + -1.0 * + pd.DataFrame( + find_difference_relative_to_comparison_series( + num_deaths.loc[0], + comparison='Baseline', + scaled=True) + ).T + ).iloc[0].unstack().reindex(param_names).drop(['Baseline']) + + num_dalys_averted = summarize( + -1.0 * + pd.DataFrame( + find_difference_relative_to_comparison_series( + num_dalys.loc[0], + comparison='Baseline') + ).T + ).iloc[0].unstack().reindex(param_names).drop(['Baseline']) + + pc_dalys_averted = 100.0 * summarize( + -1.0 * + pd.DataFrame( + find_difference_relative_to_comparison_series( + num_dalys.loc[0], + comparison='Baseline', + scaled=True) + ).T + ).iloc[0].unstack().reindex(param_names).drop(['Baseline']) + + # DEATHS + name_of_plot = f'Additional Deaths Averted vs Baseline, {target_period()}' + fig, ax = do_bar_plot_with_ci( + num_deaths_averted.clip(lower=0.0), + annotations=[ + f"{round(row['mean'], 0)} ({round(row['lower'], 1)}-{round(row['upper'], 1)}) %" + for _, row in pc_deaths_averted.clip(lower=0.0).iterrows() + ] + ) + ax.set_title(name_of_plot) + ax.set_ylabel('Additional Deaths Averted vs Baseline') + fig.tight_layout() + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + # DALYS + name_of_plot = f'DALYs Averted vs Baseline, {target_period()}' + fig, ax = do_bar_plot_with_ci( + (num_dalys_averted / 1e6).clip(lower=0.0), + annotations=[ + f"{round(row['mean'])} ({round(row['lower'], 1)}-{round(row['upper'], 1)}) %" + for _, row in pc_dalys_averted.clip(lower=0.0).iterrows() + ] + ) + ax.set_title(name_of_plot) + ax.set_ylabel('Additional DALYS Averted vs Baseline \n(Millions)') + fig.tight_layout() + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + + # %% DALYS averted relative to Baseline - broken down by major cause (HIV, TB, MALARIA) + + def get_total_num_dalys_by_label(_df): + """Return the total number of DALYS in the TARGET_PERIOD by wealth and cause label.""" + y = _df \ + .loc[_df['year'].between(*[d.year for d in TARGET_PERIOD])] \ + .drop(columns=['date', 'year', 'li_wealth']) \ + .sum(axis=0) + + # define course cause mapper for HIV, TB, MALARIA and OTHER + causes = { + 'AIDS': 'HIV/AIDS', + 'TB (non-AIDS)': 'TB', + 'Malaria': 'Malaria', + '': 'Other', # defined in order to use this dict to determine ordering of the causes in output + } + causes_relabels = y.index.map(causes).fillna('Other') + + return y.groupby(by=causes_relabels).sum()[list(causes.values())] + + total_num_dalys_by_label_results = extract_results( + results_folder, + module="tlo.methods.healthburden", + key="dalys_by_wealth_stacked_by_age_and_time", + custom_generate_series=get_total_num_dalys_by_label, + do_scaling=True, + ).pipe(set_param_names_as_column_index_level_0) + + total_num_dalys_by_label_results_averted_vs_baseline = summarize( + -1.0 * find_difference_relative_to_comparison_series_dataframe( + total_num_dalys_by_label_results, + comparison='Baseline' + ), + only_mean=True + ) + + # Check that when we sum across the causes, we get the same total as calculated when we didn't split by cause. + assert ( + (total_num_dalys_by_label_results_averted_vs_baseline.sum(axis=0).sort_index() + - num_dalys_averted['mean'].sort_index() + ) < 1e-6 + ).all() + + # Make a separate plot for the scale-up of each program/programs + plots = { + 'HIV programs': [ + 'HIV Programs Scale-up WITHOUT HSS PACKAGE', + 'HIV Programs Scale-up WITH HSS PACKAGE', + ], + 'TB programs': [ + 'TB Programs Scale-up WITHOUT HSS PACKAGE', + 'TB Programs Scale-up WITH HSS PACKAGE', + ], + 'Malaria programs': [ + 'Malaria Programs Scale-up WITHOUT HSS PACKAGE', + 'Malaria Programs Scale-up WITH HSS PACKAGE', + ], + 'All programs': [ + 'FULL HSS PACKAGE', + 'HIV/Tb/Malaria Programs Scale-up WITHOUT HSS PACKAGE', + 'HIV/Tb/Malaria Programs Scale-up WITH HSS PACKAGE', + ] + } + + for plot_name, scenario_names in plots.items(): + name_of_plot = f'{plot_name}' + fig, ax = plt.subplots() + total_num_dalys_by_label_results_averted_vs_baseline[scenario_names].T.plot.bar( + stacked=True, + ax=ax, + rot=0, + alpha=0.75 + ) + ax.set_ylim([0, 10e7]) + ax.set_title(name_of_plot) + ax.set_ylabel(f'DALYs Averted vs Baseline, {target_period()}\n(Millions)') + wrapped_labs = ["\n".join(textwrap.wrap(_lab.get_text(), 20)) for _lab in ax.get_xticklabels()] + ax.set_xticklabels(wrapped_labs) + fig.tight_layout() + fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', ''))) + fig.show() + plt.close(fig) + + # todo: Neaten graphs + # todo: other metrics of health + # todo: other graphs, broken down by age/sex (this can also be cribbed from overview paper stuff) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("results_folder", type=Path) # outputs/horizontal_and_vertical_programs-2024-05-16 + args = parser.parse_args() + + apply( + results_folder=args.results_folder, + output_folder=args.results_folder, + resourcefilepath=Path('./resources') + ) diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/mini_analysis_for_testing/mini_version_scenario.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/mini_analysis_for_testing/mini_version_scenario.py new file mode 100644 index 0000000000..a991c019ba --- /dev/null +++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/mini_analysis_for_testing/mini_version_scenario.py @@ -0,0 +1,109 @@ +"""This Scenario file is intended to help with debugging the scale-up of HIV. Tb and Malaria services, per issue #1413. + +Changes to the main analysis: + +* We're running this in MODE 1 and we're only looking. +* We're capturing the logged output from HIV, Tb and malaria +* We're limiting it to few scenarios: baseline + the scale-up of all HTM programs (no HealthSystem scale-up) + +""" + +from pathlib import Path +from typing import Dict + +from scripts.comparison_of_horizontal_and_vertical_programs.scenario_definitions import ( + ScenarioDefinitions, +) +from tlo import Date, logging +from tlo.analysis.utils import get_parameters_for_status_quo, mix_scenarios +from tlo.methods.fullmodel import fullmodel +from tlo.methods.scenario_switcher import ImprovedHealthSystemAndCareSeekingScenarioSwitcher +from tlo.scenario import BaseScenario + + +class MiniRunHTMWithAndWithoutHSS(BaseScenario): + def __init__(self): + super().__init__() + self.seed = 0 + self.start_date = Date(2010, 1, 1) + self.end_date = Date(2031, 1, 1) + self.pop_size = 100_000 + self._scenarios = self._get_scenarios() + self.number_of_draws = len(self._scenarios) + self.runs_per_draw = 1 + + def log_configuration(self): + return { + 'filename': 'mini_htm_with_and_without_hss', + 'directory': Path('./outputs'), + 'custom_levels': { + '*': logging.WARNING, + 'tlo.methods.demography': logging.INFO, + 'tlo.methods.demography.detail': logging.WARNING, + 'tlo.methods.healthburden': logging.INFO, + 'tlo.methods.healthsystem': logging.WARNING, + 'tlo.methods.healthsystem.summary': logging.INFO, + 'tlo.methods.hiv': logging.INFO, + 'tlo.methods.tb': logging.INFO, + 'tlo.methods.malaria': logging.INFO, + } + } + + def modules(self): + return ( + fullmodel(resourcefilepath=self.resources) + + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)] + ) + + def draw_parameters(self, draw_number, rng): + if draw_number < len(self._scenarios): + return list(self._scenarios.values())[draw_number] + + def _get_scenarios(self) -> Dict[str, Dict]: + """Return the Dict with values for the parameters that are changed, keyed by a name for the scenario.""" + # Load helper class containing the definitions of the elements of all the scenarios + scenario_definitions = ScenarioDefinitions() + + return { + "Baseline": + self._baseline(), + + # - - - HIV & TB & MALARIA SCALE-UP WITHOUT HSS PACKAGE- - - + "HIV/Tb/Malaria Programs Scale-up WITHOUT HSS PACKAGE": + mix_scenarios( + scenario_definitions._baseline(), + scenario_definitions._hiv_scaleup(), + scenario_definitions._tb_scaleup(), + scenario_definitions._malaria_scaleup(), + ), + } + + def _baseline(self): + self.YEAR_OF_CHANGE_FOR_HSS = 2019 + + return mix_scenarios( + get_parameters_for_status_quo(), + { + "HealthSystem": { + "mode_appt_constraints": 1, # <-- Mode 1 prior to change to preserve calibration + "mode_appt_constraints_postSwitch": 1, # <-- ***** NO CHANGE --- STAYING IN MODE 1 + "scale_to_effective_capabilities": False, # <-- irrelevant, as not changing mode + "year_mode_switch": 2100, # <-- irrelevant as not changing modes + + # Baseline scenario is with absence of HCW + 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE_FOR_HSS, + 'HR_scaling_by_level_and_officer_type_mode': 'with_absence', + + # Normalize the behaviour of Mode 2 (irrelevant as in Mode 1) + "policy_name": "Naive", + "tclose_overwrite": 1, + "tclose_days_offset_overwrite": 7, + } + }, + ) + + +if __name__ == '__main__': + from tlo.cli import scenario_run + + scenario_run([__file__]) diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py new file mode 100644 index 0000000000..6465b57f49 --- /dev/null +++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py @@ -0,0 +1,90 @@ +"""The file contains all the definitions of scenarios used the Horizontal and Vertical Program Impact Analyses""" +from typing import Dict + +from tlo.analysis.utils import get_parameters_for_status_quo, mix_scenarios + + +class ScenarioDefinitions: + + @property + def YEAR_OF_CHANGE_FOR_HSS(self) -> int: + """Year in which Health Systems Strengthening changes are made.""" + return 2019 # <-- baseline year of Human Resources for Health is 2018, and this is consistent with calibration + # during 2015-2019 period. + + + @property + def YEAR_OF_CHANGE_FOR_HTM(self) -> int: + """Year in which HIV, TB, Malaria scale-up changes are made.""" + return 2019 # todo <-- what is the natural year of scale-up? Should this be the same as the when the HSS + # changes happen? + + def _baseline(self) -> Dict: + """Return the Dict with values for the parameter changes that define the baseline scenario. """ + return mix_scenarios( + get_parameters_for_status_quo(), + { + "HealthSystem": { + "mode_appt_constraints": 1, # <-- Mode 1 prior to change to preserve calibration + "mode_appt_constraints_postSwitch": 2, # <-- Mode 2 post-change to show effects of HRH + "scale_to_effective_capabilities": True, + # <-- Transition into Mode2 with the effective capabilities in HRH 'revealed' in Mode 1 + "year_mode_switch": self.YEAR_OF_CHANGE_FOR_HSS, + + # Baseline scenario is with absence of HCW + 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE_FOR_HSS, + 'HR_scaling_by_level_and_officer_type_mode': 'with_absence', + # todo <-- Do we want the first part of the run be with_abscence too...? (Although that will mean + # that there is actually greater capacity if we do the rescaling) + + # Normalize the behaviour of Mode 2 + "policy_name": "Naive", + "tclose_overwrite": 1, + "tclose_days_offset_overwrite": 7, + } + }, + ) + + def _hss_package(self) -> Dict: + """The parameters for the Health System Strengthening Package""" + return { + 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': { + 'max_healthsystem_function': [False, True], # <-- switch from False to True mid-way + 'max_healthcare_seeking': [False, True], # <-- switch from False to True mid-way + 'year_of_switch': self.YEAR_OF_CHANGE_FOR_HSS + }, + 'HealthSystem': { + 'year_cons_availability_switch': self.YEAR_OF_CHANGE_FOR_HSS, + 'cons_availability_postSwitch': 'all', + 'yearly_HR_scaling_mode': 'GDP_growth_fHE_case5', + 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE_FOR_HSS, + 'HR_scaling_by_level_and_officer_type_mode': 'no_absence_&_x2_fac0+1', + } + } + + def _hiv_scaleup(self) -> Dict: + """The parameters for the scale-up of the HIV program""" + return { + "Hiv": { + 'do_scaleup': True, + 'scaleup_start_year': self.YEAR_OF_CHANGE_FOR_HTM, + } + } + + def _tb_scaleup(self) -> Dict: + """The parameters for the scale-up of the TB program""" + return { + "Tb": { + 'do_scaleup': True, + 'scaleup_start_year': self.YEAR_OF_CHANGE_FOR_HTM, + } + } + + def _malaria_scaleup(self) -> Dict: + """The parameters for the scale-up of the Malaria program""" + return { + 'Malaria': { + 'do_scaleup': True, + 'scaleup_start_year': self.YEAR_OF_CHANGE_FOR_HTM, + } + } diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_hss_elements.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_hss_elements.py new file mode 100644 index 0000000000..e1aebec8f6 --- /dev/null +++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_hss_elements.py @@ -0,0 +1,242 @@ +"""This Scenario file run the model under different assumptions for the HealthSystem and Vertical Program Scale-up + +Run on the batch system using: +``` +tlo batch-submit + src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_hss_elements.py +``` + +""" + +from pathlib import Path +from typing import Dict + +# from scripts.comparison_of_horizontal_and_vertical_programs.scenario_definitions import ( +# ScenarioDefinitions, +# ) +from tlo import Date, logging +from tlo.analysis.utils import get_parameters_for_status_quo, mix_scenarios +from tlo.methods.fullmodel import fullmodel +from tlo.methods.scenario_switcher import ImprovedHealthSystemAndCareSeekingScenarioSwitcher +from tlo.scenario import BaseScenario + + +class HSSElements(BaseScenario): + def __init__(self): + super().__init__() + self.seed = 0 + self.start_date = Date(2010, 1, 1) + self.end_date = Date(2031, 1, 1) + self.pop_size = 100_000 + self._scenarios = self._get_scenarios() + self.number_of_draws = len(self._scenarios) + self.runs_per_draw = 3 # <--- todo: N.B. Very small number of repeated run, to be efficient for now + + def log_configuration(self): + return { + 'filename': 'hss_elements', + 'directory': Path('./outputs'), + 'custom_levels': { + '*': logging.WARNING, + 'tlo.methods.demography': logging.INFO, + 'tlo.methods.demography.detail': logging.WARNING, + 'tlo.methods.healthburden': logging.INFO, + 'tlo.methods.healthsystem': logging.WARNING, + 'tlo.methods.healthsystem.summary': logging.INFO, + } + } + + def modules(self): + return ( + fullmodel(resourcefilepath=self.resources) + + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)] + ) + + def draw_parameters(self, draw_number, rng): + if draw_number < len(self._scenarios): + return list(self._scenarios.values())[draw_number] + + def _get_scenarios(self) -> Dict[str, Dict]: + """Return the Dict with values for the parameters that are changed, keyed by a name for the scenario.""" + # todo - decide on final definition of scenarios and the scenario package + # todo - refactorize to use the ScenariosDefinitions helperclass, which will make sure that this script and + # 'scenario_vertical_programs)_with_and_without_hss.py' are synchronised (e.g. baseline and HSS pkg scenarios) + + self.YEAR_OF_CHANGE = 2019 + # <-- baseline year of Human Resources for Health is 2018, and this is consistent with calibration during + # 2015-2019 period. + + return { + "Baseline": self._baseline(), + + # *************************** + # HEALTH SYSTEM STRENGTHENING + # *************************** + + # - - - Human Resource for Health - - - + + "Reduced Absence": + mix_scenarios( + self._baseline(), + { + 'HealthSystem': { + 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE, + 'HR_scaling_by_level_and_officer_type_mode': 'no_absence', + } + } + ), + + "Reduced Absence + Double Capacity at Primary Care": + mix_scenarios( + self._baseline(), + { + 'HealthSystem': { + 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE, + 'HR_scaling_by_level_and_officer_type_mode': 'no_absence_&_x2_fac0+1', + } + } + ), + + "HRH Keeps Pace with Population Growth": + mix_scenarios( + self._baseline(), + { + 'HealthSystem': { + 'yearly_HR_scaling_mode': 'scaling_by_population_growth', + # This is in-line with population growth _after 2018_ (baseline year for HRH) + } + } + ), + + "HRH Increases at GDP Growth": + mix_scenarios( + self._baseline(), + { + 'HealthSystem': { + 'yearly_HR_scaling_mode': 'GDP_growth', + # This is GDP growth after 2018 (baseline year for HRH) + } + } + ), + + "HRH Increases above GDP Growth": + mix_scenarios( + self._baseline(), + { + 'HealthSystem': { + 'yearly_HR_scaling_mode': 'GDP_growth_fHE_case5', + # This is above-GDP growth after 2018 (baseline year for HRH) + } + } + ), + + + # - - - Quality of Care - - - + "Perfect Clinical Practice": + mix_scenarios( + self._baseline(), + { + 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': { + 'max_healthsystem_function': [False, True], # <-- switch from False to True mid-way + 'year_of_switch': self.YEAR_OF_CHANGE, + } + }, + ), + + "Perfect Healthcare Seeking": + mix_scenarios( + get_parameters_for_status_quo(), + { + 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': { + 'max_healthcare_seeking': [False, True], + 'year_of_switch': self.YEAR_OF_CHANGE, + } + }, + ), + + # - - - Supply Chains - - - + "Perfect Availability of Vital Items": + mix_scenarios( + self._baseline(), + { + 'HealthSystem': { + 'year_cons_availability_switch': self.YEAR_OF_CHANGE, + 'cons_availability_postSwitch': 'all_vital_available', + } + } + ), + + "Perfect Availability of Medicines": + mix_scenarios( + self._baseline(), + { + 'HealthSystem': { + 'year_cons_availability_switch': self.YEAR_OF_CHANGE, + 'cons_availability_postSwitch': 'all_medicines_available', + } + } + ), + + "Perfect Availability of All Consumables": + mix_scenarios( + self._baseline(), + { + 'HealthSystem': { + 'year_cons_availability_switch': self.YEAR_OF_CHANGE, + 'cons_availability_postSwitch': 'all', + } + } + ), + + # - - - FULL PACKAGE OF HEALTH SYSTEM STRENGTHENING - - - + "FULL PACKAGE": + mix_scenarios( + self._baseline(), + { + 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': { + 'max_healthsystem_function': [False, True], # <-- switch from False to True mid-way + 'max_healthcare_seeking': [False, True], # <-- switch from False to True mid-way + 'year_of_switch': self.YEAR_OF_CHANGE + }, + 'HealthSystem': { + 'year_cons_availability_switch': self.YEAR_OF_CHANGE, + 'cons_availability_postSwitch': 'all', + 'yearly_HR_scaling_mode': 'GDP_growth_fHE_case5', + 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE, + 'HR_scaling_by_level_and_officer_type_mode': 'no_absence_&_x2_fac0+1', + } + }, + ), + + } + + def _baseline(self) -> Dict: + """Return the Dict with values for the parameter changes that define the baseline scenario. """ + return mix_scenarios( + get_parameters_for_status_quo(), + { + "HealthSystem": { + "mode_appt_constraints": 1, # <-- Mode 1 prior to change to preserve calibration + "mode_appt_constraints_postSwitch": 2, # <-- Mode 2 post-change to show effects of HRH + "scale_to_effective_capabilities": True, # <-- Transition into Mode2 with the effective + # capabilities in HRH 'revealed' in Mode 1 + "year_mode_switch": self.YEAR_OF_CHANGE, + + # Baseline scenario is with absence of HCW + 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE, + 'HR_scaling_by_level_and_officer_type_mode': 'with_absence', + # todo <-- Do we want the first part of the run be with_abscence too...? (Although that will mean + # that there is actually greater capacity if we do the rescaling) + + # Normalize the behaviour of Mode 2 + "policy_name": "Naive", + "tclose_overwrite": 1, + "tclose_days_offset_overwrite": 7, + } + }, + ) + +if __name__ == '__main__': + from tlo.cli import scenario_run + + scenario_run([__file__]) diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_vertical_programs_with_and_without_hss.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_vertical_programs_with_and_without_hss.py new file mode 100644 index 0000000000..ce7f664fe9 --- /dev/null +++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_vertical_programs_with_and_without_hss.py @@ -0,0 +1,144 @@ +"""This Scenario file run the model under different assumptions for the HealthSystem and Vertical Program Scale-up + +Run on the batch system using: +``` +tlo batch-submit + src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_vertical_programs_with_and_without_hss.py +``` + +""" + +from pathlib import Path +from typing import Dict + +from scripts.comparison_of_horizontal_and_vertical_programs.scenario_definitions import ( + ScenarioDefinitions, +) +from tlo import Date, logging +from tlo.analysis.utils import mix_scenarios +from tlo.methods.fullmodel import fullmodel +from tlo.methods.scenario_switcher import ImprovedHealthSystemAndCareSeekingScenarioSwitcher +from tlo.scenario import BaseScenario + + +class HTMWithAndWithoutHSS(BaseScenario): + def __init__(self): + super().__init__() + self.seed = 0 + self.start_date = Date(2010, 1, 1) + self.end_date = Date(2031, 1, 1) + self.pop_size = 100_000 + self._scenarios = self._get_scenarios() + self.number_of_draws = len(self._scenarios) + self.runs_per_draw = 3 # <--- todo: N.B. Very small number of repeated run, to be efficient for now + + def log_configuration(self): + return { + 'filename': 'htm_with_and_without_hss', + 'directory': Path('./outputs'), + 'custom_levels': { + '*': logging.WARNING, + 'tlo.methods.demography': logging.INFO, + 'tlo.methods.demography.detail': logging.WARNING, + 'tlo.methods.healthburden': logging.INFO, + 'tlo.methods.healthsystem': logging.WARNING, + 'tlo.methods.healthsystem.summary': logging.INFO, + } + } + + def modules(self): + return ( + fullmodel(resourcefilepath=self.resources) + + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)] + ) + + def draw_parameters(self, draw_number, rng): + if draw_number < len(self._scenarios): + return list(self._scenarios.values())[draw_number] + + def _get_scenarios(self) -> Dict[str, Dict]: + """Return the Dict with values for the parameters that are changed, keyed by a name for the scenario.""" + # Load helper class containing the definitions of the elements of all the scenarios + scenario_definitions = ScenarioDefinitions() + + return { + "Baseline": + scenario_definitions._baseline(), + + # - - - FULL PACKAGE OF HEALTH SYSTEM STRENGTHENING - - - + "FULL HSS PACKAGE": + mix_scenarios( + scenario_definitions._baseline(), + scenario_definitions._hss_package(), + ), + + # ************************************************** + # VERTICAL PROGRAMS WITH AND WITHOUT THE HSS PACKAGE + # ************************************************** + + # - - - HIV SCALE-UP WITHOUT HSS PACKAGE- - - + "HIV Programs Scale-up WITHOUT HSS PACKAGE": + mix_scenarios( + scenario_definitions._baseline(), + scenario_definitions._hiv_scaleup(), + ), + # - - - HIV SCALE-UP *WITH* HSS PACKAGE- - - + "HIV Programs Scale-up WITH HSS PACKAGE": + mix_scenarios( + scenario_definitions._baseline(), + scenario_definitions._hiv_scaleup(), + scenario_definitions._hss_package(), + ), + + # - - - TB SCALE-UP WITHOUT HSS PACKAGE- - - + "TB Programs Scale-up WITHOUT HSS PACKAGE": + mix_scenarios( + scenario_definitions._baseline(), + scenario_definitions._tb_scaleup(), + ), + # - - - TB SCALE-UP *WITH* HSS PACKAGE- - - + "TB Programs Scale-up WITH HSS PACKAGE": + mix_scenarios( + scenario_definitions._baseline(), + scenario_definitions._tb_scaleup(), + scenario_definitions._hss_package(), + ), + + # - - - MALARIA SCALE-UP WITHOUT HSS PACKAGE- - - + "Malaria Programs Scale-up WITHOUT HSS PACKAGE": + mix_scenarios( + scenario_definitions._baseline(), + scenario_definitions._malaria_scaleup(), + ), + # - - - MALARIA SCALE-UP *WITH* HSS PACKAGE- - - + "Malaria Programs Scale-up WITH HSS PACKAGE": + mix_scenarios( + scenario_definitions._baseline(), + scenario_definitions._malaria_scaleup(), + scenario_definitions._hss_package(), + ), + + # - - - HIV & TB & MALARIA SCALE-UP WITHOUT HSS PACKAGE- - - + "HIV/Tb/Malaria Programs Scale-up WITHOUT HSS PACKAGE": + mix_scenarios( + scenario_definitions._baseline(), + scenario_definitions._hiv_scaleup(), + scenario_definitions._tb_scaleup(), + scenario_definitions._malaria_scaleup(), + ), + # - - - HIV & TB & MALARIA SCALE-UP *WITH* HSS PACKAGE- - - + "HIV/Tb/Malaria Programs Scale-up WITH HSS PACKAGE": + mix_scenarios( + scenario_definitions._baseline(), + scenario_definitions._hiv_scaleup(), + scenario_definitions._tb_scaleup(), + scenario_definitions._malaria_scaleup(), + scenario_definitions._hss_package(), + ), + } + + +if __name__ == '__main__': + from tlo.cli import scenario_run + + scenario_run([__file__]) From c147e2bfbf715e262f38411269776582d14273a4 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:50:13 +0100 Subject: [PATCH 14/19] edit ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx (#1433) --- ...urceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx b/resources/ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx index 8fc0a24ae9..7ec045407a 100644 --- a/resources/ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx +++ b/resources/ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1b462c20ca6cbf0ca1f98936416e015fa248289e5bf4f66838e1b9920874f651 -size 48142 +oid sha256:49282122ff1c60e3bf73765013b6770a2fbd3f0df9a6e3f71e1d4c40e9cdfa2a +size 48238 From 965b69d67add740ff868d175bf7d7bfc1cdda4b8 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 29 Jul 2024 14:52:38 +0100 Subject: [PATCH 15/19] Updates to the Horizontal and Vertical Programs Analysis (#1439) * remove the scenario of reduced abscence and the notion in the baseline of their being any abscence * in the ResourceFile_HR_scaling_by_level_and_officer_type.xlsx: * Remove the sheet to do with absence * Rename the sheet for the scale-up scenario to remove any mention of absence * remove todo comment * refactor so that we have definition of baseline, each hss element, and the hss package in one place; and that the hss package doesn't involve duplicates of the definition of each element * add loggers for HIV, TB, Malaria in the scenario_vertical_programs_with_and_without_hss.py * add test to confirm behavioru of healthsystem scaleup parameters * linting * isort --- ..._HR_scaling_by_level_and_officer_type.xlsx | 4 +- .../mini_version_scenario.py | 36 +--- .../scenario_definitions.py | 102 +++++++++--- .../scenario_hss_elements.py | 155 ++++-------------- ..._vertical_programs_with_and_without_hss.py | 57 ++++--- tests/test_healthsystem.py | 53 ++++++ 6 files changed, 200 insertions(+), 207 deletions(-) diff --git a/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_HR_scaling_by_level_and_officer_type.xlsx b/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_HR_scaling_by_level_and_officer_type.xlsx index 1dc4b6ea7e..e7f34296e6 100644 --- a/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_HR_scaling_by_level_and_officer_type.xlsx +++ b/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_HR_scaling_by_level_and_officer_type.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:80651d157772a292bf9617c86e2616d8165a20385ada6d85a5244aca9c55aa0c -size 21938 +oid sha256:5f6e1a0c8ec505dd613dfc9c0b1b14d16ee3161500bc08c743398754d2074203 +size 15682 diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/mini_analysis_for_testing/mini_version_scenario.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/mini_analysis_for_testing/mini_version_scenario.py index a991c019ba..24256efd3a 100644 --- a/src/scripts/comparison_of_horizontal_and_vertical_programs/mini_analysis_for_testing/mini_version_scenario.py +++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/mini_analysis_for_testing/mini_version_scenario.py @@ -15,7 +15,7 @@ ScenarioDefinitions, ) from tlo import Date, logging -from tlo.analysis.utils import get_parameters_for_status_quo, mix_scenarios +from tlo.analysis.utils import mix_scenarios from tlo.methods.fullmodel import fullmodel from tlo.methods.scenario_switcher import ImprovedHealthSystemAndCareSeekingScenarioSwitcher from tlo.scenario import BaseScenario @@ -66,42 +66,18 @@ def _get_scenarios(self) -> Dict[str, Dict]: return { "Baseline": - self._baseline(), + scenario_definitions.baseline(), # - - - HIV & TB & MALARIA SCALE-UP WITHOUT HSS PACKAGE- - - "HIV/Tb/Malaria Programs Scale-up WITHOUT HSS PACKAGE": mix_scenarios( - scenario_definitions._baseline(), - scenario_definitions._hiv_scaleup(), - scenario_definitions._tb_scaleup(), - scenario_definitions._malaria_scaleup(), + scenario_definitions.baseline(), + scenario_definitions.hiv_scaleup(), + scenario_definitions.tb_scaleup(), + scenario_definitions.malaria_scaleup(), ), } - def _baseline(self): - self.YEAR_OF_CHANGE_FOR_HSS = 2019 - - return mix_scenarios( - get_parameters_for_status_quo(), - { - "HealthSystem": { - "mode_appt_constraints": 1, # <-- Mode 1 prior to change to preserve calibration - "mode_appt_constraints_postSwitch": 1, # <-- ***** NO CHANGE --- STAYING IN MODE 1 - "scale_to_effective_capabilities": False, # <-- irrelevant, as not changing mode - "year_mode_switch": 2100, # <-- irrelevant as not changing modes - - # Baseline scenario is with absence of HCW - 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE_FOR_HSS, - 'HR_scaling_by_level_and_officer_type_mode': 'with_absence', - - # Normalize the behaviour of Mode 2 (irrelevant as in Mode 1) - "policy_name": "Naive", - "tclose_overwrite": 1, - "tclose_days_offset_overwrite": 7, - } - }, - ) - if __name__ == '__main__': from tlo.cli import scenario_run diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py index 6465b57f49..8cea0f5f5a 100644 --- a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py +++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py @@ -12,17 +12,17 @@ def YEAR_OF_CHANGE_FOR_HSS(self) -> int: return 2019 # <-- baseline year of Human Resources for Health is 2018, and this is consistent with calibration # during 2015-2019 period. - @property def YEAR_OF_CHANGE_FOR_HTM(self) -> int: """Year in which HIV, TB, Malaria scale-up changes are made.""" - return 2019 # todo <-- what is the natural year of scale-up? Should this be the same as the when the HSS - # changes happen? + return 2019 - def _baseline(self) -> Dict: + def baseline(self) -> Dict: """Return the Dict with values for the parameter changes that define the baseline scenario. """ return mix_scenarios( - get_parameters_for_status_quo(), + get_parameters_for_status_quo(), # <-- Parameters that have been the calibration targets + + # Set up the HealthSystem to transition from Mode 1 -> Mode 2, with rescaling when there are HSS changes { "HealthSystem": { "mode_appt_constraints": 1, # <-- Mode 1 prior to change to preserve calibration @@ -31,12 +31,6 @@ def _baseline(self) -> Dict: # <-- Transition into Mode2 with the effective capabilities in HRH 'revealed' in Mode 1 "year_mode_switch": self.YEAR_OF_CHANGE_FOR_HSS, - # Baseline scenario is with absence of HCW - 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE_FOR_HSS, - 'HR_scaling_by_level_and_officer_type_mode': 'with_absence', - # todo <-- Do we want the first part of the run be with_abscence too...? (Although that will mean - # that there is actually greater capacity if we do the rescaling) - # Normalize the behaviour of Mode 2 "policy_name": "Naive", "tclose_overwrite": 1, @@ -45,24 +39,90 @@ def _baseline(self) -> Dict: }, ) - def _hss_package(self) -> Dict: - """The parameters for the Health System Strengthening Package""" + def double_capacity_at_primary_care(self) -> Dict: + return { + 'HealthSystem': { + 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE_FOR_HSS, + 'HR_scaling_by_level_and_officer_type_mode': 'x2_fac0&1', + } + } + + def hrh_at_pop_grwoth(self) -> Dict: + return { + 'HealthSystem': { + 'yearly_HR_scaling_mode': 'scaling_by_population_growth', + # This is in-line with population growth _after 2018_ (baseline year for HRH) + } + } + + def hrh_at_gdp_growth(self) -> Dict: + return { + 'HealthSystem': { + 'yearly_HR_scaling_mode': 'GDP_growth', + # This is GDP growth after 2018 (baseline year for HRH) + } + } + + def hrh_above_gdp_growth(self) -> Dict: + return { + 'HealthSystem': { + 'yearly_HR_scaling_mode': 'GDP_growth_fHE_case5', + # This is above-GDP growth after 2018 (baseline year for HRH) + } + } + + def perfect_clinical_practices(self) -> Dict: return { 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': { 'max_healthsystem_function': [False, True], # <-- switch from False to True mid-way + 'year_of_switch': self.YEAR_OF_CHANGE_FOR_HSS, + } + } + + def perfect_healthcare_seeking(self) -> Dict: + return { + 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': { 'max_healthcare_seeking': [False, True], # <-- switch from False to True mid-way - 'year_of_switch': self.YEAR_OF_CHANGE_FOR_HSS - }, + 'year_of_switch': self.YEAR_OF_CHANGE_FOR_HSS, + } + } + + def vital_items_available(self) -> Dict: + return { + 'HealthSystem': { + 'year_cons_availability_switch': self.YEAR_OF_CHANGE_FOR_HSS, + 'cons_availability_postSwitch': 'all_vital_available', + } + } + + def medicines_available(self) -> Dict: + return { + 'HealthSystem': { + 'year_cons_availability_switch': self.YEAR_OF_CHANGE_FOR_HSS, + 'cons_availability_postSwitch': 'all_medicines_available', + } + } + + def all_consumables_available(self) -> Dict: + return { 'HealthSystem': { 'year_cons_availability_switch': self.YEAR_OF_CHANGE_FOR_HSS, 'cons_availability_postSwitch': 'all', - 'yearly_HR_scaling_mode': 'GDP_growth_fHE_case5', - 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE_FOR_HSS, - 'HR_scaling_by_level_and_officer_type_mode': 'no_absence_&_x2_fac0+1', } } - def _hiv_scaleup(self) -> Dict: + def hss_package(self) -> Dict: + """The parameters for the Health System Strengthening Package""" + return mix_scenarios( + self.double_capacity_at_primary_care(), # } + self.hrh_above_gdp_growth(), # } <-- confirmed that these two do build on one another under + # mode 2 rescaling: see `test_scaling_up_HRH_using_yearly_scaling_and_scaling_by_level_together`. + self.perfect_clinical_practices(), + self.perfect_healthcare_seeking(), + self.all_consumables_available(), + ) + + def hiv_scaleup(self) -> Dict: """The parameters for the scale-up of the HIV program""" return { "Hiv": { @@ -71,7 +131,7 @@ def _hiv_scaleup(self) -> Dict: } } - def _tb_scaleup(self) -> Dict: + def tb_scaleup(self) -> Dict: """The parameters for the scale-up of the TB program""" return { "Tb": { @@ -80,7 +140,7 @@ def _tb_scaleup(self) -> Dict: } } - def _malaria_scaleup(self) -> Dict: + def malaria_scaleup(self) -> Dict: """The parameters for the scale-up of the Malaria program""" return { 'Malaria': { diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_hss_elements.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_hss_elements.py index e1aebec8f6..8c2f2afc09 100644 --- a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_hss_elements.py +++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_hss_elements.py @@ -11,11 +11,11 @@ from pathlib import Path from typing import Dict -# from scripts.comparison_of_horizontal_and_vertical_programs.scenario_definitions import ( -# ScenarioDefinitions, -# ) +from scripts.comparison_of_horizontal_and_vertical_programs.scenario_definitions import ( + ScenarioDefinitions, +) from tlo import Date, logging -from tlo.analysis.utils import get_parameters_for_status_quo, mix_scenarios +from tlo.analysis.utils import mix_scenarios from tlo.methods.fullmodel import fullmodel from tlo.methods.scenario_switcher import ImprovedHealthSystemAndCareSeekingScenarioSwitcher from tlo.scenario import BaseScenario @@ -58,16 +58,11 @@ def draw_parameters(self, draw_number, rng): def _get_scenarios(self) -> Dict[str, Dict]: """Return the Dict with values for the parameters that are changed, keyed by a name for the scenario.""" - # todo - decide on final definition of scenarios and the scenario package - # todo - refactorize to use the ScenariosDefinitions helperclass, which will make sure that this script and - # 'scenario_vertical_programs)_with_and_without_hss.py' are synchronised (e.g. baseline and HSS pkg scenarios) - self.YEAR_OF_CHANGE = 2019 - # <-- baseline year of Human Resources for Health is 2018, and this is consistent with calibration during - # 2015-2019 period. + scenario_definitions = ScenarioDefinitions() return { - "Baseline": self._baseline(), + "Baseline": scenario_definitions.baseline(), # *************************** # HEALTH SYSTEM STRENGTHENING @@ -75,166 +70,72 @@ def _get_scenarios(self) -> Dict[str, Dict]: # - - - Human Resource for Health - - - - "Reduced Absence": + "Double Capacity at Primary Care": mix_scenarios( - self._baseline(), - { - 'HealthSystem': { - 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE, - 'HR_scaling_by_level_and_officer_type_mode': 'no_absence', - } - } - ), - - "Reduced Absence + Double Capacity at Primary Care": - mix_scenarios( - self._baseline(), - { - 'HealthSystem': { - 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE, - 'HR_scaling_by_level_and_officer_type_mode': 'no_absence_&_x2_fac0+1', - } - } + scenario_definitions.baseline(), + scenario_definitions.double_capacity_at_primary_care(), ), "HRH Keeps Pace with Population Growth": mix_scenarios( - self._baseline(), - { - 'HealthSystem': { - 'yearly_HR_scaling_mode': 'scaling_by_population_growth', - # This is in-line with population growth _after 2018_ (baseline year for HRH) - } - } + scenario_definitions.baseline(), + scenario_definitions._hrh_at_pop_growth(), ), "HRH Increases at GDP Growth": mix_scenarios( - self._baseline(), - { - 'HealthSystem': { - 'yearly_HR_scaling_mode': 'GDP_growth', - # This is GDP growth after 2018 (baseline year for HRH) - } - } + scenario_definitions.baseline(), + scenario_definitions._hrh_at_grp_growth(), ), "HRH Increases above GDP Growth": mix_scenarios( - self._baseline(), - { - 'HealthSystem': { - 'yearly_HR_scaling_mode': 'GDP_growth_fHE_case5', - # This is above-GDP growth after 2018 (baseline year for HRH) - } - } + scenario_definitions.baseline(), + scenario_definitions.hrh_above_gdp_growth(), ), # - - - Quality of Care - - - "Perfect Clinical Practice": mix_scenarios( - self._baseline(), - { - 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': { - 'max_healthsystem_function': [False, True], # <-- switch from False to True mid-way - 'year_of_switch': self.YEAR_OF_CHANGE, - } - }, + scenario_definitions.baseline(), + scenario_definitions._perfect_clinical_practice(), ), "Perfect Healthcare Seeking": mix_scenarios( - get_parameters_for_status_quo(), - { - 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': { - 'max_healthcare_seeking': [False, True], - 'year_of_switch': self.YEAR_OF_CHANGE, - } - }, + scenario_definitions.baseline(), + scenario_definitions.perfect_healthcare_seeking(), ), # - - - Supply Chains - - - "Perfect Availability of Vital Items": mix_scenarios( - self._baseline(), - { - 'HealthSystem': { - 'year_cons_availability_switch': self.YEAR_OF_CHANGE, - 'cons_availability_postSwitch': 'all_vital_available', - } - } + scenario_definitions.baseline(), + scenario_definitions.vital_items_available(), ), "Perfect Availability of Medicines": mix_scenarios( - self._baseline(), - { - 'HealthSystem': { - 'year_cons_availability_switch': self.YEAR_OF_CHANGE, - 'cons_availability_postSwitch': 'all_medicines_available', - } - } + scenario_definitions.baseline(), + scenario_definitions.medicines_available(), + ), "Perfect Availability of All Consumables": mix_scenarios( - self._baseline(), - { - 'HealthSystem': { - 'year_cons_availability_switch': self.YEAR_OF_CHANGE, - 'cons_availability_postSwitch': 'all', - } - } + scenario_definitions.baseline(), + scenario_definitions.all_consumables_available(), ), # - - - FULL PACKAGE OF HEALTH SYSTEM STRENGTHENING - - - "FULL PACKAGE": mix_scenarios( - self._baseline(), - { - 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': { - 'max_healthsystem_function': [False, True], # <-- switch from False to True mid-way - 'max_healthcare_seeking': [False, True], # <-- switch from False to True mid-way - 'year_of_switch': self.YEAR_OF_CHANGE - }, - 'HealthSystem': { - 'year_cons_availability_switch': self.YEAR_OF_CHANGE, - 'cons_availability_postSwitch': 'all', - 'yearly_HR_scaling_mode': 'GDP_growth_fHE_case5', - 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE, - 'HR_scaling_by_level_and_officer_type_mode': 'no_absence_&_x2_fac0+1', - } - }, + scenario_definitions.baseline(), + scenario_definitions.hss_package(), ), - } - def _baseline(self) -> Dict: - """Return the Dict with values for the parameter changes that define the baseline scenario. """ - return mix_scenarios( - get_parameters_for_status_quo(), - { - "HealthSystem": { - "mode_appt_constraints": 1, # <-- Mode 1 prior to change to preserve calibration - "mode_appt_constraints_postSwitch": 2, # <-- Mode 2 post-change to show effects of HRH - "scale_to_effective_capabilities": True, # <-- Transition into Mode2 with the effective - # capabilities in HRH 'revealed' in Mode 1 - "year_mode_switch": self.YEAR_OF_CHANGE, - - # Baseline scenario is with absence of HCW - 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE, - 'HR_scaling_by_level_and_officer_type_mode': 'with_absence', - # todo <-- Do we want the first part of the run be with_abscence too...? (Although that will mean - # that there is actually greater capacity if we do the rescaling) - - # Normalize the behaviour of Mode 2 - "policy_name": "Naive", - "tclose_overwrite": 1, - "tclose_days_offset_overwrite": 7, - } - }, - ) if __name__ == '__main__': from tlo.cli import scenario_run diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_vertical_programs_with_and_without_hss.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_vertical_programs_with_and_without_hss.py index ce7f664fe9..e4f6dcbd88 100644 --- a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_vertical_programs_with_and_without_hss.py +++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_vertical_programs_with_and_without_hss.py @@ -43,6 +43,9 @@ def log_configuration(self): 'tlo.methods.healthburden': logging.INFO, 'tlo.methods.healthsystem': logging.WARNING, 'tlo.methods.healthsystem.summary': logging.INFO, + 'tlo.methods.hiv': logging.INFO, + 'tlo.methods.tb': logging.INFO, + 'tlo.methods.malaria': logging.INFO, } } @@ -63,13 +66,13 @@ def _get_scenarios(self) -> Dict[str, Dict]: return { "Baseline": - scenario_definitions._baseline(), + scenario_definitions.baseline(), # - - - FULL PACKAGE OF HEALTH SYSTEM STRENGTHENING - - - "FULL HSS PACKAGE": mix_scenarios( - scenario_definitions._baseline(), - scenario_definitions._hss_package(), + scenario_definitions.baseline(), + scenario_definitions.hss_package(), ), # ************************************************** @@ -79,61 +82,61 @@ def _get_scenarios(self) -> Dict[str, Dict]: # - - - HIV SCALE-UP WITHOUT HSS PACKAGE- - - "HIV Programs Scale-up WITHOUT HSS PACKAGE": mix_scenarios( - scenario_definitions._baseline(), - scenario_definitions._hiv_scaleup(), + scenario_definitions.baseline(), + scenario_definitions.hiv_scaleup(), ), # - - - HIV SCALE-UP *WITH* HSS PACKAGE- - - "HIV Programs Scale-up WITH HSS PACKAGE": mix_scenarios( - scenario_definitions._baseline(), - scenario_definitions._hiv_scaleup(), - scenario_definitions._hss_package(), + scenario_definitions.baseline(), + scenario_definitions.hiv_scaleup(), + scenario_definitions.hss_package(), ), # - - - TB SCALE-UP WITHOUT HSS PACKAGE- - - "TB Programs Scale-up WITHOUT HSS PACKAGE": mix_scenarios( - scenario_definitions._baseline(), - scenario_definitions._tb_scaleup(), + scenario_definitions.baseline(), + scenario_definitions.tb_scaleup(), ), # - - - TB SCALE-UP *WITH* HSS PACKAGE- - - "TB Programs Scale-up WITH HSS PACKAGE": mix_scenarios( - scenario_definitions._baseline(), - scenario_definitions._tb_scaleup(), - scenario_definitions._hss_package(), + scenario_definitions.baseline(), + scenario_definitions.tb_scaleup(), + scenario_definitions.hss_package(), ), # - - - MALARIA SCALE-UP WITHOUT HSS PACKAGE- - - "Malaria Programs Scale-up WITHOUT HSS PACKAGE": mix_scenarios( - scenario_definitions._baseline(), - scenario_definitions._malaria_scaleup(), + scenario_definitions.baseline(), + scenario_definitions.malaria_scaleup(), ), # - - - MALARIA SCALE-UP *WITH* HSS PACKAGE- - - "Malaria Programs Scale-up WITH HSS PACKAGE": mix_scenarios( - scenario_definitions._baseline(), - scenario_definitions._malaria_scaleup(), - scenario_definitions._hss_package(), + scenario_definitions.baseline(), + scenario_definitions.malaria_scaleup(), + scenario_definitions.hss_package(), ), # - - - HIV & TB & MALARIA SCALE-UP WITHOUT HSS PACKAGE- - - "HIV/Tb/Malaria Programs Scale-up WITHOUT HSS PACKAGE": mix_scenarios( - scenario_definitions._baseline(), - scenario_definitions._hiv_scaleup(), - scenario_definitions._tb_scaleup(), - scenario_definitions._malaria_scaleup(), + scenario_definitions.baseline(), + scenario_definitions.hiv_scaleup(), + scenario_definitions.tb_scaleup(), + scenario_definitions.malaria_scaleup(), ), # - - - HIV & TB & MALARIA SCALE-UP *WITH* HSS PACKAGE- - - "HIV/Tb/Malaria Programs Scale-up WITH HSS PACKAGE": mix_scenarios( - scenario_definitions._baseline(), - scenario_definitions._hiv_scaleup(), - scenario_definitions._tb_scaleup(), - scenario_definitions._malaria_scaleup(), - scenario_definitions._hss_package(), + scenario_definitions.baseline(), + scenario_definitions.hiv_scaleup(), + scenario_definitions.tb_scaleup(), + scenario_definitions.malaria_scaleup(), + scenario_definitions.hss_package(), ), } diff --git a/tests/test_healthsystem.py b/tests/test_healthsystem.py index ae212a4f48..6568d1df56 100644 --- a/tests/test_healthsystem.py +++ b/tests/test_healthsystem.py @@ -2517,3 +2517,56 @@ def run_sim(dynamic_HR_scaling_factor: Dict[int, float]) -> tuple: ratio_in_sim = caps / initial_caps assert np.allclose(ratio_in_sim, expected_overall_scaling) + + +def test_scaling_up_HRH_using_yearly_scaling_and_scaling_by_level_together(seed): + """We want the behaviour of HRH 'yearly scaling' and 'scaling_by_level' to operate together, so that, for instance, + the total capabilities is greater when scaling up by level _and_ by yearly-scaling than by using either + independently.""" + + def get_capabilities(yearly_scaling: bool, scaling_by_level: bool, rescaling: bool) -> float: + """Return total capabilities of HRH when optionally using 'yearly scaling' and/or 'scaling_by_level'""" + sim = Simulation(start_date=start_date, seed=seed) + sim.register( + demography.Demography(resourcefilepath=resourcefilepath), + healthsystem.HealthSystem(resourcefilepath=resourcefilepath), + simplified_births.SimplifiedBirths(resourcefilepath=resourcefilepath), + ) + params = sim.modules['HealthSystem'].parameters + + # In Mode 1, from the beginning. + params["mode_appt_constraints"] = 1 + + if yearly_scaling: + params['yearly_HR_scaling_mode'] = 'GDP_growth_fHE_case5' + # This is above-GDP growth after 2018 (baseline year for HRH) + + if scaling_by_level: + params['year_HR_scaling_by_level_and_officer_type'] = 2018 # <-- same time as yearly-scaling + params['HR_scaling_by_level_and_officer_type_mode'] = 'x2_fac0&1' + + if rescaling: + # Switch to Mode 2, with the rescaling, at the same time as the other changes occur + params["mode_appt_constraints_postSwitch"] = 2 + params["scale_to_effective_capabilities"] = True + params["year_mode_switch"] = 2018 + + popsize = 100 + sim.make_initial_population(n=popsize) + sim.simulate(end_date=sim.date + pd.DateOffset(years=10, days=1)) # run simulation until at least past 2018 + + return sim.modules['HealthSystem'].capabilities_today.sum() + + # - When running without any rescaling + caps_only_scaling_by_level = get_capabilities(yearly_scaling=False, scaling_by_level=True, rescaling=False) + caps_only_scaling_by_year = get_capabilities(yearly_scaling=True, scaling_by_level=False, rescaling=False) + caps_scaling_by_both = get_capabilities(yearly_scaling=True, scaling_by_level=True, rescaling=False) + assert caps_scaling_by_both > caps_only_scaling_by_level + assert caps_scaling_by_both > caps_only_scaling_by_year + + # - When there is also rescaling as we go from Mode 2 into Mode 1 + caps_only_scaling_by_level_with_rescaling = get_capabilities(yearly_scaling=False, scaling_by_level=True, rescaling=True) + caps_only_scaling_by_year_with_rescaling = get_capabilities(yearly_scaling=True, scaling_by_level=False, rescaling=True) + caps_scaling_by_both_with_rescaling = get_capabilities(yearly_scaling=True, scaling_by_level=True, rescaling=True) + assert caps_scaling_by_both_with_rescaling > caps_only_scaling_by_level_with_rescaling + assert caps_scaling_by_both_with_rescaling > caps_only_scaling_by_year_with_rescaling From 9561a7a220bd0701e4e881447e945700a4393406 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Mon, 29 Jul 2024 15:29:59 +0100 Subject: [PATCH 16/19] Let the Scale-up of HIV, TB, Malaria services go to "MAXIMAL" levels (as well as target levels) (#1443) * create new scenario file for testing HTM max scale-up update ResourceFile_HIV.xlsx, ResourceFile_malaria.xlsx, ResourceFile_TB.xlsx to include max scale-up * change parameter do_sacleup to categorical in hiv, tb and malaria modules * add type_of_scaleup with default value=none to htm resourcefiles * choose scale-up values from resourcefile using parameter type_of_scaleup * set up scenarios to test max scale-up works as expected * set up scenarios to test max scale-up works as expected * edit figures * switch type_of_scaleup to string * update test_htm_scaleup.py * enhanced_lifestly.py update np.timedelta64 as ValueError: Unit M is not supported. Only unambiguous timedelta values durations are supported. Allowed units are 'W', 'D', 'h', 'm', 's', 'ms', 'us', 'ns' * switch scale-up parameter selection (target vs max) to update_parameters_for_program_scaleup not within read_parameters as this is read before the analysis script updates parameters * comment out plots in analysis_maxHTM_scenario.py * tidy up script * move import statements used for plotting * remove unneeded columns from scale-up_parameters sheet in resourcefiles * switch to using maximum scale-up * roll back change in enhanced_lifestyle.py --------- Co-authored-by: tdm32 --- resources/ResourceFile_HIV.xlsx | 4 +- resources/ResourceFile_TB.xlsx | 4 +- resources/malaria/ResourceFile_malaria.xlsx | 4 +- .../analysis_maxHTM_scenario.py | 229 ++++++++++++++++++ .../scenario_definitions.py | 6 +- src/tlo/methods/hiv.py | 20 +- src/tlo/methods/malaria.py | 23 +- src/tlo/methods/tb.py | 20 +- tests/test_htm_scaleup.py | 42 ++-- 9 files changed, 299 insertions(+), 53 deletions(-) create mode 100644 src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py diff --git a/resources/ResourceFile_HIV.xlsx b/resources/ResourceFile_HIV.xlsx index b2db25c898..00f7b684db 100644 --- a/resources/ResourceFile_HIV.xlsx +++ b/resources/ResourceFile_HIV.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f57142efaf515d74f8290238ce1abad7b99871f9195623112892b3bb535bf634 -size 161721 +oid sha256:b34a88635b02ee8a465462c8eb67a485d721c9159a5bba1df8e63609b803ebe9 +size 161679 diff --git a/resources/ResourceFile_TB.xlsx b/resources/ResourceFile_TB.xlsx index 3dfc69cd81..2b612ad6ec 100644 --- a/resources/ResourceFile_TB.xlsx +++ b/resources/ResourceFile_TB.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:120d687122772909c267db41c933664ccc6247c8aef59d49532547c0c3791121 -size 55634 +oid sha256:93d7bf76c8bece548e08e3f0cb6e9e28a09ca2b5760a408399bf9641f7ed2001 +size 56523 diff --git a/resources/malaria/ResourceFile_malaria.xlsx b/resources/malaria/ResourceFile_malaria.xlsx index a6487e80ae..7537f3ace9 100644 --- a/resources/malaria/ResourceFile_malaria.xlsx +++ b/resources/malaria/ResourceFile_malaria.xlsx @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e8157368754dae9ce692fbd10fecf1e598f37fb258292085c93e1c881dd47aa9 -size 69590 +oid sha256:7f256d5007b36e2428ae844747bd766bb6086540c5135408d606dd821e185d9f +size 69578 diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py new file mode 100644 index 0000000000..0cfcd05315 --- /dev/null +++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py @@ -0,0 +1,229 @@ +""" +This scenario file sets up the scenarios for simulating the effects of scaling up programs + +The scenarios are: +*0 baseline mode 1 +*1 scale-up HIV program +*2 scale-up TB program +*3 scale-up malaria program +*4 scale-up HIV and Tb and malaria programs + +scale-up occurs on the default scale-up start date (01/01/2025: in parameters list of resourcefiles) + +For all scenarios, keep all default health system settings + +check the batch configuration gets generated without error: +tlo scenario-run --draw-only src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py + +Run on the batch system using: +tlo batch-submit src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py + +or locally using: +tlo scenario-run src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py + +or execute a single run: +tlo scenario-run src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py --draw 1 0 + +""" + +import datetime +from pathlib import Path + +from tlo import Date, logging +from tlo.methods import ( + demography, + enhanced_lifestyle, + epi, + healthburden, + healthseekingbehaviour, + healthsystem, + hiv, + malaria, + simplified_births, + symptommanager, + tb, +) +from tlo.scenario import BaseScenario + +resourcefilepath = Path("./resources") +datestamp = datetime.date.today().strftime("__%Y_%m_%d") + +outputspath = Path("./outputs") +scaleup_start_year = 2012 +end_date = Date(2015, 1, 1) + + +class EffectOfProgrammes(BaseScenario): + def __init__(self): + super().__init__() + self.seed = 0 + self.start_date = Date(2010, 1, 1) + self.end_date = end_date + self.pop_size = 1_000 + self.number_of_draws = 5 + self.runs_per_draw = 1 + + def log_configuration(self): + return { + 'filename': 'scaleup_tests', + 'directory': Path('./outputs'), # <- (specified only for local running) + 'custom_levels': { + '*': logging.WARNING, + 'tlo.methods.hiv': logging.INFO, + 'tlo.methods.tb': logging.INFO, + 'tlo.methods.malaria': logging.INFO, + 'tlo.methods.demography': logging.INFO, + } + } + + def modules(self): + return [ + demography.Demography(resourcefilepath=self.resources), + simplified_births.SimplifiedBirths(resourcefilepath=self.resources), + enhanced_lifestyle.Lifestyle(resourcefilepath=self.resources), + healthsystem.HealthSystem(resourcefilepath=self.resources), + symptommanager.SymptomManager(resourcefilepath=self.resources), + healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=self.resources), + healthburden.HealthBurden(resourcefilepath=self.resources), + epi.Epi(resourcefilepath=self.resources), + hiv.Hiv(resourcefilepath=self.resources), + tb.Tb(resourcefilepath=self.resources), + malaria.Malaria(resourcefilepath=self.resources), + ] + + def draw_parameters(self, draw_number, rng): + + return { + 'Hiv': { + 'type_of_scaleup': ['none', 'max', 'none', 'none', 'max'][draw_number], + 'scaleup_start_year': scaleup_start_year, + }, + 'Tb': { + 'type_of_scaleup': ['none', 'none', 'max', 'none', 'max'][draw_number], + 'scaleup_start_year': scaleup_start_year, + }, + 'Malaria': { + 'type_of_scaleup': ['none', 'none', 'none', 'max', 'max'][draw_number], + 'scaleup_start_year': scaleup_start_year, + }, + } + + +if __name__ == '__main__': + from tlo.cli import scenario_run + + scenario_run([__file__]) + + + +# %% Produce some figures and summary info + +# import pandas as pd +# import matplotlib.pyplot as plt + +# # Find results_folder associated with a given batch_file (and get most recent [-1]) +# results_folder = get_scenario_outputs("scaleup_tests-", outputspath)[-1] +# +# # get basic information about the results +# info = get_scenario_info(results_folder) +# +# # 1) Extract the parameters that have varied over the set of simulations +# params = extract_params(results_folder) +# +# +# # DEATHS +# +# +# def get_num_deaths_by_cause_label(_df): +# """Return total number of Deaths by label within the TARGET_PERIOD +# values are summed for all ages +# df returned: rows=COD, columns=draw +# """ +# return _df \ +# .loc[pd.to_datetime(_df.date).between(*TARGET_PERIOD)] \ +# .groupby(_df['label']) \ +# .size() +# +# +# TARGET_PERIOD = (Date(scaleup_start_year, 1, 1), end_date) +# +# # produce df of total deaths over scale-up period +# num_deaths_by_cause_label = extract_results( +# results_folder, +# module='tlo.methods.demography', +# key='death', +# custom_generate_series=get_num_deaths_by_cause_label, +# do_scaling=True +# ) +# +# +# def summarise_deaths_for_one_cause(results_folder, label): +# """ returns mean deaths for each year of the simulation +# values are aggregated across the runs of each draw +# for the specified cause +# """ +# +# results_deaths = extract_results( +# results_folder, +# module="tlo.methods.demography", +# key="death", +# custom_generate_series=( +# lambda df: df.assign(year=df["date"].dt.year).groupby( +# ["year", "label"])["person_id"].count() +# ), +# do_scaling=True, +# ) +# # removes multi-index +# results_deaths = results_deaths.reset_index() +# +# # select only cause specified +# tmp = results_deaths.loc[ +# (results_deaths.label == label) +# ] +# +# # group deaths by year +# tmp = pd.DataFrame(tmp.groupby(["year"]).sum()) +# +# # get mean for each draw +# mean_deaths = pd.concat({'mean': tmp.iloc[:, 1:].groupby(level=0, axis=1).mean()}, axis=1).swaplevel(axis=1) +# +# return mean_deaths +# +# +# aids_deaths = summarise_deaths_for_one_cause(results_folder, 'AIDS') +# tb_deaths = summarise_deaths_for_one_cause(results_folder, 'TB (non-AIDS)') +# malaria_deaths = summarise_deaths_for_one_cause(results_folder, 'Malaria') +# +# +# draw_labels = ['No scale-up', 'HIV scale-up', 'TB scale-up', 'Malaria scale-up', 'HTM scale-up'] +# colours = ['blue', 'green', 'red', 'purple', 'orange'] +# +# # Create subplots +# fig, axs = plt.subplots(3, 1, figsize=(10, 10)) +# # Plot for df1 +# for i, col in enumerate(aids_deaths.columns): +# axs[0].plot(aids_deaths.index, aids_deaths[col], label=draw_labels[i], +# color=colours[i]) +# axs[0].set_title('HIV/AIDS') +# axs[0].legend(loc='center left', bbox_to_anchor=(1, 0.5)) # Legend to the right of the plot +# axs[0].axvline(x=scaleup_start_year, color='gray', linestyle='--') +# +# # Plot for df2 +# for i, col in enumerate(tb_deaths.columns): +# axs[1].plot(tb_deaths.index, tb_deaths[col], color=colours[i]) +# axs[1].set_title('TB') +# axs[1].axvline(x=scaleup_start_year, color='gray', linestyle='--') +# +# # Plot for df3 +# for i, col in enumerate(malaria_deaths.columns): +# axs[2].plot(malaria_deaths.index, malaria_deaths[col], color=colours[i]) +# axs[2].set_title('Malaria') +# axs[2].axvline(x=scaleup_start_year, color='gray', linestyle='--') +# +# for ax in axs: +# ax.set_xlabel('Years') +# ax.set_ylabel('Number deaths') +# +# plt.tight_layout(rect=[0, 0, 0.85, 1]) # Adjust layout to make space for legend +# plt.show() +# diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py index 8cea0f5f5a..31615bdc27 100644 --- a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py +++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py @@ -126,7 +126,7 @@ def hiv_scaleup(self) -> Dict: """The parameters for the scale-up of the HIV program""" return { "Hiv": { - 'do_scaleup': True, + 'type_of_scaleup': 'max', # <--- using MAXIMUM SCALE-UP as an experiment 'scaleup_start_year': self.YEAR_OF_CHANGE_FOR_HTM, } } @@ -135,7 +135,7 @@ def tb_scaleup(self) -> Dict: """The parameters for the scale-up of the TB program""" return { "Tb": { - 'do_scaleup': True, + 'type_of_scaleup': 'max', # <--- using MAXIMUM SCALE-UP as an experiment 'scaleup_start_year': self.YEAR_OF_CHANGE_FOR_HTM, } } @@ -144,7 +144,7 @@ def malaria_scaleup(self) -> Dict: """The parameters for the scale-up of the Malaria program""" return { 'Malaria': { - 'do_scaleup': True, + 'type_of_scaleup': 'max', # <--- using MAXIMUM SCALE-UP as an experiment 'scaleup_start_year': self.YEAR_OF_CHANGE_FOR_HTM, } } diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py index d744c9d254..d6455cc861 100644 --- a/src/tlo/methods/hiv.py +++ b/src/tlo/methods/hiv.py @@ -398,16 +398,16 @@ def __init__(self, name=None, resourcefilepath=None, run_with_checks=False): " high-bound-exclusive]", ), # ------------------ scale-up parameters for scenario analysis ------------------ # - "do_scaleup": Parameter( - Types.BOOL, - "argument to determine whether scale-up of program will be implemented" + "type_of_scaleup": Parameter( + Types.STRING, "argument to determine type scale-up of program which will be implemented, " + "can be 'none', 'target' or 'max'", ), "scaleup_start_year": Parameter( Types.INT, "the year when the scale-up starts (it will occur on 1st January of that year)" ), "scaleup_parameters": Parameter( - Types.DICT, + Types.DATA_FRAME, "the parameters and values changed in scenario analysis" ), } @@ -448,7 +448,7 @@ def read_parameters(self, data_folder): p["treatment_cascade"] = workbook["spectrum_treatment_cascade"] # load parameters for scale-up projections - p["scaleup_parameters"] = workbook["scaleup_parameters"].set_index('parameter')['scaleup_value'].to_dict() + p['scaleup_parameters'] = workbook["scaleup_parameters"] # DALY weights # get the DALY weight that this module will use from the weight database (these codes are just random!) @@ -914,7 +914,7 @@ def initialise_simulation(self, sim): sim.schedule_event(HivLoggingEvent(self), sim.date + DateOffset(years=1)) # Optional: Schedule the scale-up of programs - if self.parameters["do_scaleup"]: + if self.parameters["type_of_scaleup"] != 'none': scaleup_start_date = Date(self.parameters["scaleup_start_year"], 1, 1) assert scaleup_start_date >= self.sim.start_date, f"Date {scaleup_start_date} is before simulation starts." sim.schedule_event(HivScaleUpEvent(self), scaleup_start_date) @@ -1102,8 +1102,14 @@ def initialise_simulation(self, sim): ) def update_parameters_for_program_scaleup(self): + """ options for program scale-up are 'target' or 'max' """ p = self.parameters - scaled_params = p["scaleup_parameters"] + scaled_params_workbook = p["scaleup_parameters"] + + if p['type_of_scaleup'] == 'target': + scaled_params = scaled_params_workbook.set_index('parameter')['target_value'].to_dict() + else: + scaled_params = scaled_params_workbook.set_index('parameter')['max_value'].to_dict() # scale-up HIV program # reduce risk of HIV - applies to whole adult population diff --git a/src/tlo/methods/malaria.py b/src/tlo/methods/malaria.py index a5e9738a99..b1fdfb09dd 100644 --- a/src/tlo/methods/malaria.py +++ b/src/tlo/methods/malaria.py @@ -189,17 +189,16 @@ def __init__(self, name=None, resourcefilepath=None): Types.REAL, 'probability that treatment will clear malaria symptoms' ), - # ------------------ scale-up parameters for scenario analysis ------------------ # - "do_scaleup": Parameter( - Types.BOOL, - "argument to determine whether scale-up of program will be implemented" + "type_of_scaleup": Parameter( + Types.STRING, "argument to determine type scale-up of program which will be implemented, " + "can be 'none', 'target' or 'max'", ), "scaleup_start_year": Parameter( Types.INT, "the year when the scale-up starts (it will occur on 1st January of that year)" ), "scaleup_parameters": Parameter( - Types.DICT, + Types.DATA_FRAME, "the parameters and values changed in scenario analysis" ) } @@ -261,7 +260,7 @@ def read_parameters(self, data_folder): p['sev_inc'] = pd.read_csv(self.resourcefilepath / 'malaria' / 'ResourceFile_malaria_SevInc_expanded.csv') # load parameters for scale-up projections - p["scaleup_parameters"] = workbook["scaleup_parameters"].set_index('parameter')['scaleup_value'].to_dict() + p['scaleup_parameters'] = workbook["scaleup_parameters"] # check itn projected values are <=0.7 and rounded to 1dp for matching to incidence tables p['itn'] = round(p['itn'], 1) @@ -602,7 +601,7 @@ def initialise_simulation(self, sim): sim.schedule_event(MalariaPrevDistrictLoggingEvent(self), sim.date + DateOffset(months=1)) # Optional: Schedule the scale-up of programs - if self.parameters["do_scaleup"]: + if self.parameters["type_of_scaleup"] != 'none': scaleup_start_date = Date(self.parameters["scaleup_start_year"], 1, 1) assert scaleup_start_date >= self.sim.start_date, f"Date {scaleup_start_date} is before simulation starts." sim.schedule_event(MalariaScaleUpEvent(self), scaleup_start_date) @@ -659,8 +658,14 @@ def initialise_simulation(self, sim): ) def update_parameters_for_program_scaleup(self): + """ options for program scale-up are 'target' or 'max' """ p = self.parameters - scaled_params = p["scaleup_parameters"] + scaled_params_workbook = p["scaleup_parameters"] + + if p['type_of_scaleup'] == 'target': + scaled_params = scaled_params_workbook.set_index('parameter')['target_value'].to_dict() + else: + scaled_params = scaled_params_workbook.set_index('parameter')['max_value'].to_dict() # scale-up malaria program # increase testing @@ -1164,7 +1169,7 @@ def apply(self, person_id, squeeze_factor): facility_level=self.ACCEPTED_FACILITY_LEVEL, treatment_id=self.TREATMENT_ID, ) - + logger.info(key='rdt_log', data=person_details_for_test) # if positive, refer for a confirmatory test at level 1a diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py index 3392d90b62..623ee2e483 100644 --- a/src/tlo/methods/tb.py +++ b/src/tlo/methods/tb.py @@ -377,16 +377,16 @@ def __init__(self, name=None, resourcefilepath=None, run_with_checks=False): "length of inpatient stay for end-of-life TB patients", ), # ------------------ scale-up parameters for scenario analysis ------------------ # - "do_scaleup": Parameter( - Types.BOOL, - "argument to determine whether scale-up of program will be implemented" + "type_of_scaleup": Parameter( + Types.STRING, "argument to determine type scale-up of program which will be implemented, " + "can be 'none', 'target' or 'max'", ), "scaleup_start_year": Parameter( Types.INT, "the year when the scale-up starts (it will occur on 1st January of that year)" ), "scaleup_parameters": Parameter( - Types.DICT, + Types.DATA_FRAME, "the parameters and values changed in scenario analysis" ) } @@ -427,7 +427,7 @@ def read_parameters(self, data_folder): ) # load parameters for scale-up projections - p["scaleup_parameters"] = workbook["scaleup_parameters"].set_index('parameter')['scaleup_value'].to_dict() + p['scaleup_parameters'] = workbook["scaleup_parameters"] # 2) Get the DALY weights if "HealthBurden" in self.sim.modules.keys(): @@ -871,7 +871,7 @@ def initialise_simulation(self, sim): # 2) log at the end of the year # Optional: Schedule the scale-up of programs - if self.parameters["do_scaleup"]: + if self.parameters["type_of_scaleup"] != 'none': scaleup_start_date = Date(self.parameters["scaleup_start_year"], 1, 1) assert scaleup_start_date >= self.sim.start_date, f"Date {scaleup_start_date} is before simulation starts." sim.schedule_event(TbScaleUpEvent(self), scaleup_start_date) @@ -889,8 +889,14 @@ def initialise_simulation(self, sim): ) def update_parameters_for_program_scaleup(self): + """ options for program scale-up are 'target' or 'max' """ p = self.parameters - scaled_params = p["scaleup_parameters"] + scaled_params_workbook = p["scaleup_parameters"] + + if p['type_of_scaleup'] == 'target': + scaled_params = scaled_params_workbook.set_index('parameter')['target_value'].to_dict() + else: + scaled_params = scaled_params_workbook.set_index('parameter')['max_value'].to_dict() # scale-up TB program # use NTP treatment rates diff --git a/tests/test_htm_scaleup.py b/tests/test_htm_scaleup.py index dbb2638c88..fcb538f19c 100644 --- a/tests/test_htm_scaleup.py +++ b/tests/test_htm_scaleup.py @@ -84,7 +84,7 @@ def test_hiv_scale_up(seed): check_initial_params(sim) # update parameters to instruct there to be a scale-up - sim.modules["Hiv"].parameters["do_scaleup"] = True + sim.modules["Hiv"].parameters["type_of_scaleup"] = 'target' sim.modules["Hiv"].parameters["scaleup_start_year"] = scaleup_start_year # Make the population @@ -95,13 +95,13 @@ def test_hiv_scale_up(seed): assert sim.modules["Hiv"].parameters["beta"] < original_params.loc[ original_params.parameter_name == "beta", "value"].values[0] assert sim.modules["Hiv"].parameters["prob_prep_for_fsw_after_hiv_test"] == new_params.loc[ - new_params.parameter == "prob_prep_for_fsw_after_hiv_test", "scaleup_value"].values[0] + new_params.parameter == "prob_prep_for_fsw_after_hiv_test", "target_value"].values[0] assert sim.modules["Hiv"].parameters["prob_prep_for_agyw"] == new_params.loc[ - new_params.parameter == "prob_prep_for_agyw", "scaleup_value"].values[0] + new_params.parameter == "prob_prep_for_agyw", "target_value"].values[0] assert sim.modules["Hiv"].parameters["probability_of_being_retained_on_prep_every_3_months"] == new_params.loc[ - new_params.parameter == "probability_of_being_retained_on_prep_every_3_months", "scaleup_value"].values[0] + new_params.parameter == "probability_of_being_retained_on_prep_every_3_months", "target_value"].values[0] assert sim.modules["Hiv"].parameters["prob_circ_after_hiv_test"] == new_params.loc[ - new_params.parameter == "prob_circ_after_hiv_test", "scaleup_value"].values[0] + new_params.parameter == "prob_circ_after_hiv_test", "target_value"].values[0] # check malaria parameters unchanged mal_original_params = pd.read_excel(resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx', @@ -154,11 +154,11 @@ def test_htm_scale_up(seed): check_initial_params(sim) # update parameters - sim.modules["Hiv"].parameters["do_scaleup"] = True + sim.modules["Hiv"].parameters["type_of_scaleup"] = 'target' sim.modules["Hiv"].parameters["scaleup_start_year"] = scaleup_start_year - sim.modules["Tb"].parameters["do_scaleup"] = True + sim.modules["Tb"].parameters["type_of_scaleup"] = 'target' sim.modules["Tb"].parameters["scaleup_start_year"] = scaleup_start_year - sim.modules["Malaria"].parameters["do_scaleup"] = True + sim.modules["Malaria"].parameters["type_of_scaleup"] = 'target' sim.modules["Malaria"].parameters["scaleup_start_year"] = scaleup_start_year # Make the population @@ -169,42 +169,42 @@ def test_htm_scale_up(seed): assert sim.modules["Hiv"].parameters["beta"] < original_hiv_params.loc[ original_hiv_params.parameter_name == "beta", "value"].values[0] assert sim.modules["Hiv"].parameters["prob_prep_for_fsw_after_hiv_test"] == new_hiv_params.loc[ - new_hiv_params.parameter == "prob_prep_for_fsw_after_hiv_test", "scaleup_value"].values[0] + new_hiv_params.parameter == "prob_prep_for_fsw_after_hiv_test", "target_value"].values[0] assert sim.modules["Hiv"].parameters["prob_prep_for_agyw"] == new_hiv_params.loc[ - new_hiv_params.parameter == "prob_prep_for_agyw", "scaleup_value"].values[0] + new_hiv_params.parameter == "prob_prep_for_agyw", "target_value"].values[0] assert sim.modules["Hiv"].parameters["probability_of_being_retained_on_prep_every_3_months"] == new_hiv_params.loc[ - new_hiv_params.parameter == "probability_of_being_retained_on_prep_every_3_months", "scaleup_value"].values[0] + new_hiv_params.parameter == "probability_of_being_retained_on_prep_every_3_months", "target_value"].values[0] assert sim.modules["Hiv"].parameters["prob_circ_after_hiv_test"] == new_hiv_params.loc[ - new_hiv_params.parameter == "prob_circ_after_hiv_test", "scaleup_value"].values[0] + new_hiv_params.parameter == "prob_circ_after_hiv_test", "target_value"].values[0] # check malaria parameters changed new_mal_params = pd.read_excel(resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx', sheet_name="scaleup_parameters") assert sim.modules["Malaria"].parameters["prob_malaria_case_tests"] == new_mal_params.loc[ - new_mal_params.parameter == "prob_malaria_case_tests", "scaleup_value"].values[0] + new_mal_params.parameter == "prob_malaria_case_tests", "target_value"].values[0] assert sim.modules["Malaria"].parameters["rdt_testing_rates"]["Rate_rdt_testing"].eq(new_mal_params.loc[ - new_mal_params.parameter == "rdt_testing_rates", "scaleup_value"].values[0]).all() + new_mal_params.parameter == "rdt_testing_rates", "target_value"].values[0]).all() # some irs coverage levels should now = 1.0 assert sim.modules["Malaria"].itn_irs['irs_rate'].any() == 1.0 # itn rates for 2019 onwards assert sim.modules["Malaria"].parameters["itn"] == new_mal_params.loc[ - new_mal_params.parameter == "itn", "scaleup_value"].values[0] + new_mal_params.parameter == "itn", "target_value"].values[0] # check tb parameters changed new_tb_params = pd.read_excel(resourcefilepath / 'ResourceFile_TB.xlsx', sheet_name="scaleup_parameters") assert sim.modules["Tb"].parameters["rate_testing_active_tb"]["treatment_coverage"].eq(new_tb_params.loc[ - new_tb_params.parameter == "tb_treatment_coverage", "scaleup_value"].values[0]).all() + new_tb_params.parameter == "tb_treatment_coverage", "target_value"].values[0]).all() assert sim.modules["Tb"].parameters["prob_tx_success_ds"] == new_tb_params.loc[ - new_tb_params.parameter == "tb_prob_tx_success_ds", "scaleup_value"].values[0] + new_tb_params.parameter == "tb_prob_tx_success_ds", "target_value"].values[0] assert sim.modules["Tb"].parameters["prob_tx_success_mdr"] == new_tb_params.loc[ - new_tb_params.parameter == "tb_prob_tx_success_mdr", "scaleup_value"].values[0] + new_tb_params.parameter == "tb_prob_tx_success_mdr", "target_value"].values[0] assert sim.modules["Tb"].parameters["prob_tx_success_0_4"] == new_tb_params.loc[ - new_tb_params.parameter == "tb_prob_tx_success_0_4", "scaleup_value"].values[0] + new_tb_params.parameter == "tb_prob_tx_success_0_4", "target_value"].values[0] assert sim.modules["Tb"].parameters["prob_tx_success_5_14"] == new_tb_params.loc[ - new_tb_params.parameter == "tb_prob_tx_success_5_14", "scaleup_value"].values[0] + new_tb_params.parameter == "tb_prob_tx_success_5_14", "target_value"].values[0] assert sim.modules["Tb"].parameters["first_line_test"] == new_tb_params.loc[ - new_tb_params.parameter == "first_line_test", "scaleup_value"].values[0] + new_tb_params.parameter == "first_line_test", "target_value"].values[0] From 78d7de1b98a15ea66b24ed1f960e319a57926670 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:49:36 +0100 Subject: [PATCH 17/19] change logging level from warning to debug to prevent this message filling logs with same message millions of times. (#1409) --- src/tlo/methods/hsi_event.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tlo/methods/hsi_event.py b/src/tlo/methods/hsi_event.py index 85feb2b1b5..b76a865d2d 100644 --- a/src/tlo/methods/hsi_event.py +++ b/src/tlo/methods/hsi_event.py @@ -358,7 +358,7 @@ def _check_if_appt_footprint_can_run(self) -> bool: ): return True else: - logger.warning( + logger.debug( key="message", data=( f"The expected footprint of {self.TREATMENT_ID} is not possible with the configuration of " From d9d3f62dd37c4a4cb278497e41207c091ffab699 Mon Sep 17 00:00:00 2001 From: Tim Hallett <39991060+tbhallett@users.noreply.github.com> Date: Fri, 2 Aug 2024 16:35:46 +0100 Subject: [PATCH 18/19] Update contributors (#1450) Co-authored-by: Asif Tamuri --- CITATION.cff | 4 ++++ contributors.yaml | 40 +++++++++++++++++++++++++++++++++------- docs/tlo_contributors.py | 5 +++-- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 3d5d0c7cc0..d6849dece4 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -113,6 +113,10 @@ authors: family-names: Janoušková orcid: https://orcid.org/0000-0002-4104-0119 affiliation: University College London +- given-names: Rachel + family-names: Murray-Watson + affiliation: Imperial College London + orcid: https://orcid.org/0000-0001-9079-5975 repository-code: https://github.com/UCL/TLOmodel url: https://tlomodel.org abstract: Our fundamental aim is to develop the use of epidemiological and economic diff --git a/contributors.yaml b/contributors.yaml index 1ea698d181..75cd14f1d9 100644 --- a/contributors.yaml +++ b/contributors.yaml @@ -4,7 +4,7 @@ affiliation: "Imperial College London" website: "https://www.imperial.ac.uk/people/timothy.hallett" github-username: tbhallett - role: Joint lead epidemiology + role: Project Lead contributions: - Epidemiology and modelling - Software development @@ -14,7 +14,7 @@ affiliation: "University College London" website: "https://profiles.ucl.ac.uk/5430" github-username: andrew-phillips-1 - role: Joint lead epidemiology + role: Lead Epidemiology contributions: - Epidemiology and modelling - Software development @@ -102,7 +102,6 @@ website: "https://www.york.ac.uk/che/staff/research/sakshi-mohan/" github-username: sakshimohan contributions: - - Epidemiology and modelling - Health economics - Software development - given-names: Wingston @@ -206,15 +205,14 @@ affiliation: University College London website: "https://profiles.ucl.ac.uk/954" contributions: - - Clinical consultant + - Clinical process modelling - given-names: Paul family-names: Revill orcid: "https://orcid.org/0000-0001-8632-0600" affiliation: University of York website: "https://www.york.ac.uk/che/staff/research/paul-revill/" github-username: paulrevill - contributions: - - Health economics + role: "Lead Health-Economics" - given-names: Wiktoria family-names: Tafesse orcid: "https://orcid.org/0000-0002-0076-8285" @@ -237,7 +235,7 @@ website: "https://www.york.ac.uk/che/staff/students/newton-chagoma/" github-username: nchagoma503 contributions: - - Health economics + - Health economics - given-names: Martin family-names: Chalkley orcid: "https://orcid.org/0000-0002-1091-8259" @@ -273,3 +271,31 @@ family-names: Uwais website: "https://uk.linkedin.com/in/leila-uwais-597705142" github-username: Leila-Uwais +- given-names: Dominic + family-names: Nkhoma + affiliation: "Kamuzu University of Health Sciences" + orcid: "https://orcid.org/0000-0001-6125-6630" + contributions: + - Policy translation + website: "https://mw.linkedin.com/in/dominicnkhoma1978" +- given-names: Gerald + family-names: Manthalu + affiliation: "Department of Planning and Policy Development, Ministry of Health and Population, Lilongwe, Malawi" + orcid: "https://orcid.org/0000-0002-3501-8601" + contributions: + - Policy translation +- given-names: Rachel + family-names: Murray-Watson + affiliation: "Imperial College London" + orcid: https://orcid.org/0000-0001-9079-5975 + github-username: RachelMurray-Watson + contributions: + - Epidemiology and modelling + - Software development +- given-names: Victor + family-names: Mwapasa + orcid: "https://orcid.org/0000-0002-2748-8902" + affiliation: "Kamuzu University of Health Sciences" + website: "https://www.kuhes.ac.mw/prof-victor-mwapasa/" + contributions: + - Clinical process modelling diff --git a/docs/tlo_contributors.py b/docs/tlo_contributors.py index 680418efa5..0a26ebbbc3 100644 --- a/docs/tlo_contributors.py +++ b/docs/tlo_contributors.py @@ -98,11 +98,12 @@ def categorized_contributor_lists_html( with open(args.contributors_file_path, "r") as f: contributors = yaml.safe_load(f) contribution_categories = ( + "Clinical process modelling", "Epidemiology and modelling", "Health economics", - "Software development", - "Clinical consultant", + "Policy translation", "Project management", + "Software development", ) category_predicates = { "Scientific leads": lambda c: "lead" in c.get("role", "").lower(), From 58a2a4134d997645ce7f756bb501cef563938c1c Mon Sep 17 00:00:00 2001 From: Will Graham <32364977+willGraham01@users.noreply.github.com> Date: Tue, 20 Aug 2024 09:23:48 +0100 Subject: [PATCH 19/19] Fix over-allocation of Bed Days in #1399 (#1437) * 1st pass writing combination method * Write test that checks failure case in issue * Expand on docstring for combining footprints method * Force-cast to int when returning footprint since pd.datetime.timedelta doesn't know how to handle np.int64's * Catch bug when determining priority on each day, write test to cover this case with a 3-bed types resolution --------- Co-authored-by: Emmanuel Mnjowe <32415622+mnjowe@users.noreply.github.com> --- src/tlo/methods/bed_days.py | 58 ++++++++++++++++++++++++-- tests/test_beddays.py | 83 +++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 3 deletions(-) diff --git a/src/tlo/methods/bed_days.py b/src/tlo/methods/bed_days.py index 7adb6de60c..a47b75b16a 100644 --- a/src/tlo/methods/bed_days.py +++ b/src/tlo/methods/bed_days.py @@ -318,6 +318,60 @@ def issue_bed_days_according_to_availability(self, facility_id: int, footprint: return available_footprint + def combine_footprints_for_same_patient( + self, fp1: Dict[str, int], fp2: Dict[str, int] + ) -> Dict[str, int]: + """ + Given two footprints that are due to start on the same day, combine the two footprints by + overlaying the higher-priority bed over the lower-priority beds. + + As an example, given the footprints, + fp1 = {"bedtype1": 2, "bedtype2": 0} + fp2 = {"bedtype1": 1, "bedtype2": 6} + + where bedtype1 is higher priority than bedtype2, we expect the combined allocation to be + {"bedtype1": 2, "bedtype2": 5}. + + This is because footprints are assumed to run in the order of the bedtypes priority; so + fp2's second day of being allocated to bedtype2 is overwritten by the higher-priority + allocation to bedtype1 from fp1. The remaining 5 days are allocated to bedtype2 since + fp1 does not require a bed after the first 2 days, but fp2 does. + + :param fp1: Footprint, to be combined with the other argument. + :param pf2: Footprint, to be combined with the other argument. + """ + fp1_length = sum(days for days in fp1.values()) + fp2_length = sum(days for days in fp2.values()) + max_length = max(fp1_length, fp2_length) + + # np arrays where each entry is the priority of bed allocated by the footprint + # on that day. fp_priority[i] = priority of the bed allocated by the footprint on + # day i (where the current day is day 0). + # By default, fill with priority equal to the lowest bed priority; though all + # the values will have been explicitly overwritten after the next loop completes. + fp1_priority = np.ones((max_length,), dtype=int) * (len(self.bed_types) - 1) + fp2_priority = fp1_priority.copy() + + fp1_at = 0 + fp2_at = 0 + for priority, bed_type in enumerate(self.bed_types): + # Bed type priority is dictated by list order, so it is safe to loop here. + # We will start with the highest-priority bed type and work to the lowest + fp1_priority[fp1_at:fp1_at + fp1[bed_type]] = priority + fp1_at += fp1[bed_type] + fp2_priority[fp2_at:fp2_at + fp2[bed_type]] = priority + fp2_at += fp2[bed_type] + + # Element-wise minimum of the two priority arrays is then the bed to assign + final_priorities = np.minimum(fp1_priority, fp2_priority) + # Final footprint is then formed by converting the priorities into blocks of days + return { + # Cast to int here since pd.datetime.timedelta doesn't know what to do with + # np.int64 types + bed_type: int(sum(final_priorities == priority)) + for priority, bed_type in enumerate(self.bed_types) + } + def impose_beddays_footprint(self, person_id, footprint): """This is called to reflect that a new occupancy of bed-days should be recorded: * Cause to be reflected in the bed_tracker that an hsi_event is being run that will cause bed to be @@ -345,9 +399,7 @@ def impose_beddays_footprint(self, person_id, footprint): remaining_footprint = self.get_remaining_footprint(person_id) # combine the remaining footprint with the new footprint, with days in each bed-type running concurrently: - combo_footprint = {bed_type: max(footprint[bed_type], remaining_footprint[bed_type]) - for bed_type in self.bed_types - } + combo_footprint = self.combine_footprints_for_same_patient(footprint, remaining_footprint) # remove the old footprint and apply the combined footprint self.remove_beddays_footprint(person_id) diff --git a/tests/test_beddays.py b/tests/test_beddays.py index f3f3e7f087..224619e8b3 100644 --- a/tests/test_beddays.py +++ b/tests/test_beddays.py @@ -2,6 +2,7 @@ import copy import os from pathlib import Path +from typing import Dict import pandas as pd import pytest @@ -83,6 +84,88 @@ def test_beddays_in_isolation(tmpdir, seed): assert ([cap_bedtype1] * days_sim == tracker.values).all() +def test_beddays_allocation_resolution(tmpdir, seed): + sim = Simulation(start_date=start_date, seed=seed) + sim.register( + demography.Demography(resourcefilepath=resourcefilepath), + healthsystem.HealthSystem(resourcefilepath=resourcefilepath), + ) + + # Update BedCapacity data with a simple table: + level2_facility_ids = [128, 129, 130] # <-- the level 2 facilities for each region + # This ensures over-allocations have to be properly resolved + cap_bedtype1 = 10 + cap_bedtype2 = 10 + cap_bedtype3 = 10 + + # create a simple bed capacity dataframe + hs = sim.modules["HealthSystem"] + hs.parameters["BedCapacity"] = pd.DataFrame( + data={ + "Facility_ID": level2_facility_ids, + "bedtype1": cap_bedtype1, + "bedtype2": cap_bedtype2, + "bedtype3": cap_bedtype3, + } + ) + + sim.make_initial_population(n=100) + sim.simulate(end_date=start_date) + + # reset bed days tracker to the start_date of the simulation + hs.bed_days.initialise_beddays_tracker() + + def assert_footprint_matches_expected( + footprint: Dict[str, int], expected_footprint: Dict[str, int] + ): + """ + Asserts that two footprints are identical. + The footprint provided as the 2nd argument is assumed to be the footprint + that we want to match, and the 1st as the result of the program attempting + to resolve over-allocations. + """ + assert len(footprint) == len( + expected_footprint + ), "Bed type footprints did not return same allocations." + for bed_type, expected_days in expected_footprint.items(): + allocated_days = footprint[bed_type] + assert expected_days == allocated_days, ( + f"Bed type {bed_type} was allocated {allocated_days} upon combining, " + f"but expected it to get {expected_days}." + ) + + # Check that combining footprints for a person returns the expected output + + # SIMPLE 2-bed days case + # Test uses example fail case given in https://github.com/UCL/TLOmodel/issues/1399 + # Person p has: bedtyp1 for 2 days, bedtype2 for 0 days. + # Person p then assigned: bedtype1 for 1 days, bedtype2 for 6 days. + # EXPECT: p's footprints are combined into bedtype1 for 2 days, bedtype2 for 5 days. + existing_footprint = {"bedtype1": 2, "bedtype2": 0, "bedtype3": 0} + incoming_footprint = {"bedtype1": 1, "bedtype2": 6, "bedtype3": 0} + expected_resolution = {"bedtype1": 2, "bedtype2": 5, "bedtype3": 0} + allocated_footprint = hs.bed_days.combine_footprints_for_same_patient( + existing_footprint, incoming_footprint + ) + assert_footprint_matches_expected(allocated_footprint, expected_resolution) + + # TEST case involve 3 different bed-types. + # Person p has: bedtype1 for 2 days, then bedtype3 for 4 days. + # p is assigned: bedtype1 for 1 day, bedtype2 for 3 days, and bedtype3 for 1 day. + # EXPECT: p spends 2 days in each bedtype; + # - Day 1 needs bedtype1 for both footprints + # - Day 2 existing footprint at bedtype1 overwrites incoming at bedtype2 + # - Day 3 & 4 incoming footprint at bedtype2 overwrites existing allocation to bedtype3 + # - Day 5 both footprints want bedtype3 + # - Day 6 existing footprint needs bedtype3, whilst incoming footprint is over.s + existing_footprint = {"bedtype1": 2, "bedtype2": 0, "bedtype3": 4} + incoming_footprint = {"bedtype1": 1, "bedtype2": 3, "bedtype3": 1} + expected_resolution = {"bedtype1": 2, "bedtype2": 2, "bedtype3": 2} + allocated_footprint = hs.bed_days.combine_footprints_for_same_patient( + existing_footprint, incoming_footprint + ) + assert_footprint_matches_expected(allocated_footprint, expected_resolution) + def check_dtypes(simulation): # check types of columns df = simulation.population.props