diff --git a/.github/workflows/run-on-comment.yml b/.github/workflows/run-on-comment.yml
index 0edf974935..3fdc74b53d 100644
--- a/.github/workflows/run-on-comment.yml
+++ b/.github/workflows/run-on-comment.yml
@@ -126,7 +126,7 @@ jobs:
- name: Get comment-bot token
if: always() && steps.has_permissions.outputs.result == 'true'
id: get_comment_bot_token
- uses: peter-murray/workflow-application-token-action@8e4e6fbf6fcc8a272781d97597969d21b3812974
+ uses: peter-murray/workflow-application-token-action@dc0413987a085fa17d19df9e47d4677cf81ffef3
with:
application_id: ${{ secrets.application-id }}
application_private_key: ${{ secrets.application-private-key }}
diff --git a/.github/workflows/run-profiling.yaml b/.github/workflows/run-profiling.yaml
index a28acda1b0..af0611b074 100644
--- a/.github/workflows/run-profiling.yaml
+++ b/.github/workflows/run-profiling.yaml
@@ -163,7 +163,7 @@ jobs:
## The token provided needs contents and pages access to the target repo
## Token can be (re)generated by a member of the UCL organisation,
## the current member is the rc-softdev-admin.
- ## [17-07-2024] New token generated, will expire 10-07-2025
+ ## [10-07-2023] The current token will expire 10-07-2024
- name: Push results to profiling repository
uses: dmnemec/copy_file_to_another_repo_action@v1.1.1
env:
diff --git a/.github/workflows/tests-unpinned.yml b/.github/workflows/tests-unpinned.yml
deleted file mode 100644
index d776644eb9..0000000000
--- a/.github/workflows/tests-unpinned.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-name: Tests with unpinned dependencies
-
-on:
- schedule:
- - cron: 0 0 15 * *
-
-jobs:
- test:
- name: Run tests
- strategy:
- matrix:
- os: [ubuntu-latest, macos-latest, windows-latest]
- python-version: [3.x]
- fail-fast: false
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- with:
- lfs: true
- - name: Cache tox
- uses: actions/cache@v4
- with:
- path: .tox
- key: tox-${{hashFiles('pyproject.toml') }}
- - name: Set up Python
- uses: actions/setup-python@v5
- with:
- python-version: ${{ matrix.python-version }}
- - name: Install tox
- run: python -m pip install tox
- - name: Run tests
- run: tox -v -e py3-latest -- pytest -n auto -vv tests --skip-slow
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 5d39c44840..283a53594a 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -10,8 +10,6 @@ on:
- requirements/**
- resources/**
- src/tlo/**
- - src/scripts/profiling/scale_run.py
- - src/scripts/profiling/shared.py
- tests/**
- pyproject.toml
- tox.ini
@@ -46,10 +44,9 @@ jobs:
name: Set matrix
run: |
set -e
- shopt -s globstar
# Find all test files and generate their list in JSON format
VAR_FILES="{\"include\":["
- for file in tests/**/test_*.py; do
+ for file in tests/test_*.py; do
VAR_FILES="${VAR_FILES}{\"file\":\"${file}\"},"
done
VAR_FILES="${VAR_FILES}]}"
@@ -78,12 +75,4 @@ jobs:
tox --version
- name: Test with tox
run: |
- tox -v -e py311 -- pytest --show-capture=no -vv "${{ matrix.file }}" --junit-xml="${{ matrix.file }}.results.xml"
- - name: Generate test report
- if: always()
- uses: pmeier/pytest-results-action@fc6576eced1f411ea48ab10e917d9cfce2960e29
- with:
- path: ${{ matrix.file }}.results.xml
- summary: true
- display-options: fEX
- title: Results for ${{ matrix.file }}
+ tox -v -e py311,report -- pytest --cov --cov-report=term-missing -vv "${{ matrix.file }}"
diff --git a/.gitignore b/.gitignore
index 6805a8cd3d..cb11b17ff2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -109,6 +109,9 @@ venv.bak/
# PyCharm
.idea/
+# TLO .rst files
+docs/reference/tlo*.rst
+
# TLO configuration
tlo.conf
@@ -121,12 +124,3 @@ profiling_results/
# ignore _version.py file generated by setuptools_scm
src/**/_version.py
-
-# Generated TLO docs files
-docs/_*.rst
-docs/_*.html
-docs/hsi_events.csv
-docs/parameters.rst
-docs/reference/modules.rst
-docs/reference/tlo*.rst
-docs/resources/**/*.rst
diff --git a/CITATION.cff b/CITATION.cff
index 07d4c8801c..3d5d0c7cc0 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -113,11 +113,6 @@ authors:
family-names: Janoušková
orcid: https://orcid.org/0000-0002-4104-0119
affiliation: University College London
- website: https://profiles.ucl.ac.uk/90260
-- given-names: Rachel
- family-names: Murray-Watson
- affiliation: Imperial College London
- orcid: https://orcid.org/0000-0001-9079-5975
repository-code: https://github.com/UCL/TLOmodel
url: https://tlomodel.org
abstract: Our fundamental aim is to develop the use of epidemiological and economic
diff --git a/README.md b/README.md
index ce6afead33..eadcbd2c60 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
-
+
Thanzi la Onse model
\n")
-
- def write_epilogue(self):
- self.output("
\n")
-
- def write_entry(self, _key, _label, text):
- self.output(f"{text}\n")
-
-
-class DateSortingStyle(BaseSortingStyle):
- """Sorting style for bibliography in reverse (newest first) publication date order."""
-
- def sorting_key(self, entry):
- months = list(calendar.month_name)
- return (
- -int(entry.fields.get("year")),
- -months.index(entry.fields.get("month", "")),
- entry.fields.get("title", ""),
- )
-
-
-class LastOnlyNameStyle(BaseNameStyle):
- """Name style showing only last names and associated name particles."""
-
- def format(self, person, _abbr=False):
- return join[
- name_part(tie=True)[person.rich_prelast_names],
- name_part[person.rich_last_names],
- name_part(before=", ")[person.rich_lineage_names],
- ]
-
-
-@node
-def summarized_names(children, context, role, summarize_limit=3, **kwargs):
- """Return formatted names with et al. summarization when number exceeds specified limit."""
-
- assert not children
-
- try:
- persons = context["entry"].persons[role]
- except KeyError:
- raise FieldIsMissing(role, context["entry"])
-
- name_style = LastOnlyNameStyle()
- if len(persons) > summarize_limit:
- return words[name_style.format(persons[0]), "et al."].format_data(context)
- else:
- formatted_names = [name_style.format(person) for person in persons]
- return join(**kwargs)[formatted_names].format_data(context)
-
-
-class SummarizedStyle(UnsrtStyle):
- """
- Bibliography style showing summarized names, year, title and journal with expandable details.
-
- Not suitable for use with LaTeX backend due to use of details tags.
- """
-
- default_sorting_style = DateSortingStyle
-
- def _format_summarized_names(self, role):
- return summarized_names(role, sep=", ", sep2=" and ", last_sep=", and ")
-
- def _format_label(self, label):
- return tag("em")[f"{label}: "]
-
- def _format_details_as_table(self, details):
- return tag("table")[
- toplevel[
- *(
- tag("tr")[toplevel[tag("td")[tag("em")[key]], tag("td")[value]]]
- for key, value in details.items()
- )
- ]
- ]
-
- def _get_summary_template(self, e, type_):
- bibtex_type_to_venue_field = {"article": "journal", "misc": "publisher", "inproceedings": "booktitle"}
- venue_field = bibtex_type_to_venue_field[type_]
- url = first_of[
- optional[join["https://doi.org/", field("doi", raw=True)]],
- optional[field("url", raw=True)],
- "#",
- ]
- return href[
- url,
- sentence(sep=". ")[
- words[
- self._format_summarized_names("author"),
- optional["(", field("year"), ")"],
- ],
- self.format_title(e, "title", as_sentence=False),
- tag("em")[field(venue_field)],
- ],
- ]
-
- def _get_details_template(self, type_):
- bibtex_type_to_label = {"article": "Journal article", "misc": "Pre-print", "inproceedings": "Conference paper"}
- return self._format_details_as_table(
- {
- "Type": bibtex_type_to_label[type_],
- "DOI": optional[field("doi")],
- "Date": publication_date,
- "Authors": self.format_names("author"),
- "Abstract": field("abstract"),
- }
- )
-
- def _get_summarized_template(self, e, type_):
- summary_template = self._get_summary_template(e, type_)
- details_template = self._get_details_template(type_)
- return tag("details")[tag("summary")[summary_template], details_template]
-
- def get_article_template(self, e):
- return self._get_summarized_template(e, "article")
-
- def get_misc_template(self, e):
- return self._get_summarized_template(e, "misc")
-
- def get_inproceedings_template(self, e):
- return self._get_summarized_template(e, "inproceedings")
-
-
-def write_publications_list(stream, bibliography_data, section_names, backend, style):
- """Write bibliography data with given backend and style to a stream splitting in to sections."""
- keys_by_section = defaultdict(list)
- section_names = [name.lower() for name in section_names]
- for key, entry in bibliography_data.entries.items():
- # Section names and keywords normalized to lower case to make matching case-insensitive
- keywords = set(k.strip().lower() for k in entry.fields.get("keywords", "").split(","))
- section_names_in_keywords = keywords & set(section_names)
- if len(section_names_in_keywords) == 1:
- keys_by_section[section_names_in_keywords.pop()].append(key)
- elif len(section_names_in_keywords) == 0:
- msg = (
- f"BibTeX entry with key {key} does not have a keyword / tag corresponding to "
- f"one of section names {section_names} and so will not be included in output."
- )
- warn(msg, stacklevel=2)
- else:
- msg = (
- f"BibTeX entry with key {key} has multiple keywords / tags corresponding to "
- f"section names {section_names} and so will not be included in output."
- )
- warn(msg, stacklevel=2)
- for section_name in section_names:
- stream.write(f"{section_name.capitalize()}
\n")
- formatted_bibliography = style.format_bibliography(
- bibliography_data, keys_by_section[section_name]
- )
- backend.write_to_stream(formatted_bibliography, stream)
- stream.write("\n")
-
-
-if __name__ == "__main__":
- docs_directory = Path(__file__).parent
- parser = argparse.ArgumentParser(
- description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter
- )
- parser.add_argument(
- "--bib-file",
- type=Path,
- default=docs_directory / "publications.bib",
- help="BibTeX file containing publication details",
- )
- parser.add_argument(
- "--output-file",
- type=Path,
- default=docs_directory / "_publications_list.html",
- help="File to write publication list to in HTML format",
- )
- parser.add_argument(
- "--update-from-zotero",
- action="store_true",
- help="Update BibTeX file at path specified by --bib-file from Zotero group library",
- )
- parser.add_argument(
- "--zotero-group-id",
- default="5746396",
- help="Integer identifier for Zotero group library",
- )
- args = parser.parse_args()
- if args.update_from_zotero:
- endpoint_url = f"https://api.zotero.org/groups/{args.zotero_group_id}/items"
- # Zotero API requires maximum number of results to return (limit parameter)
- # to be explicitly specified for export formats such as bibtex and allows a
- # maximum value of 100 - if we exceed this number of publications will need
- # to switch to making multiple requests with different start indices
- response = requests.get(
- endpoint_url, params={"format": "bibtex", "limit": "100"}
- )
- if response.ok:
- with open(args.bib_file, "w") as bib_file:
- bib_file.write(response.text)
- else:
- msg = (
- f"Request to {endpoint_url} failed with status code "
- f"{response.status_code} ({response.reason})"
- )
- raise RuntimeError(msg)
- with open(args.output_file, "w") as output_file:
- write_publications_list(
- stream=output_file,
- bibliography_data=pybtex.database.parse_file(args.bib_file),
- section_names=[
- "Overview of the model",
- "Analyses using the model",
- "Healthcare seeking behaviour",
- "Healthcare provision",
- "Data Collection - Protocol and Analyses",
- "Theoretical Frameworks",
- ],
- backend=InlineHTMLBackend(),
- style=SummarizedStyle(),
- )
diff --git a/docs/write-ups/Epilepsy.docx b/docs/write-ups/Epilepsy.docx
index 344e6ad6fa..fb8b66055b 100644
--- a/docs/write-ups/Epilepsy.docx
+++ b/docs/write-ups/Epilepsy.docx
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:1f84018d4a66a782d95b057e25fee043458f907f5a9a973b6685f650c1e2be08
-size 2381944
+oid sha256:b394045e585544fdb83e8ec71993c5f42c0f50bdc8b016f6712c1f2a86994c8f
+size 2759724
diff --git a/pyproject.toml b/pyproject.toml
index ce24c3a3cc..f18a736844 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -14,6 +14,9 @@ classifiers = [
'Operating System :: OS Independent',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3 :: Only',
+ 'Programming Language :: Python :: 3.8',
+ 'Programming Language :: Python :: 3.9',
+ 'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
'Programming Language :: Python :: 3.12',
]
@@ -24,7 +27,7 @@ dependencies = [
"pyshp",
"squarify",
"numpy",
- "pandas~=2.0",
+ "pandas~=2.0.0",
"scipy",
# Avoid https://foss.heptapod.net/openpyxl/openpyxl/-/issues/1963
"openpyxl==3.1.0",
@@ -33,14 +36,12 @@ dependencies = [
"azure-identity",
"azure-keyvault",
"azure-storage-file-share",
- # For saving and loading simulation state
- "dill",
]
description = "Thanzi la Onse Epidemiology Model"
dynamic = ["version"]
license = {file = "LICENSE.txt"}
readme = "README.md"
-requires-python = ">=3.11"
+requires-python = ">=3.8"
[project.optional-dependencies]
dev = [
@@ -119,7 +120,7 @@ addopts = "-ra --strict-markers --doctest-modules --doctest-glob=*.rst --tb=shor
markers = ["group2", "slow"]
[tool.ruff]
-target-version = "py311"
+target-version = "py38"
line-length = 120
[tool.setuptools.packages.find]
diff --git a/requirements/base.txt b/requirements/base.txt
index 458aa584ea..dc44b868c6 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -56,8 +56,6 @@ cryptography==41.0.3
# pyjwt
cycler==0.11.0
# via matplotlib
-dill==0.3.8
- # via tlo (pyproject.toml)
et-xmlfile==1.1.0
# via openpyxl
fonttools==4.42.1
@@ -114,7 +112,6 @@ pyjwt[crypto]==2.8.0
# via
# adal
# msal
- # pyjwt
pyparsing==3.1.1
# via matplotlib
pyshp==2.3.1
diff --git a/requirements/dev.txt b/requirements/dev.txt
index a6e0468a19..efd4f0e3e8 100644
--- a/requirements/dev.txt
+++ b/requirements/dev.txt
@@ -1,5 +1,5 @@
#
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.8
# by the following command:
#
# pip-compile --extra=dev --output-file=requirements/dev.txt
@@ -61,9 +61,7 @@ colorama==0.4.6
contourpy==1.1.1
# via matplotlib
coverage[toml]==7.3.1
- # via
- # coverage
- # pytest-cov
+ # via pytest-cov
cryptography==41.0.3
# via
# adal
@@ -74,14 +72,14 @@ cryptography==41.0.3
# pyjwt
cycler==0.11.0
# via matplotlib
-dill==0.3.8
- # via
- # pylint
- # tlo (pyproject.toml)
+dill==0.3.7
+ # via pylint
distlib==0.3.7
# via virtualenv
et-xmlfile==1.1.0
# via openpyxl
+exceptiongroup==1.1.3
+ # via pytest
execnet==2.0.2
# via pytest-xdist
filelock==3.12.4
@@ -96,6 +94,10 @@ gitpython==3.1.36
# via tlo (pyproject.toml)
idna==3.4
# via requests
+importlib-metadata==6.8.0
+ # via build
+importlib-resources==6.1.1
+ # via matplotlib
iniconfig==2.0.0
# via pytest
isodate==0.6.1
@@ -170,7 +172,6 @@ pyjwt[crypto]==2.8.0
# via
# adal
# msal
- # pyjwt
pylint==3.0.1
# via tlo (pyproject.toml)
pyparsing==3.1.1
@@ -220,17 +221,29 @@ smmap==5.0.1
# via gitdb
squarify==0.4.3
# via tlo (pyproject.toml)
+tomli==2.0.1
+ # via
+ # build
+ # coverage
+ # pip-tools
+ # pylint
+ # pyproject-api
+ # pyproject-hooks
+ # pytest
+ # tox
tomlkit==0.12.1
# via pylint
tox==4.11.3
# via tlo (pyproject.toml)
typing-extensions==4.8.0
# via
+ # astroid
# azure-core
# azure-keyvault-certificates
# azure-keyvault-keys
# azure-keyvault-secrets
# azure-storage-file-share
+ # pylint
tzdata==2023.3
# via pandas
urllib3==2.0.4
@@ -241,6 +254,10 @@ virtualenv==20.24.5
# tox
wheel==0.41.2
# via pip-tools
+zipp==3.17.0
+ # via
+ # importlib-metadata
+ # importlib-resources
# The following packages are considered to be unsafe in a requirements file:
# pip
diff --git a/resources/ResourceFile_HIV.xlsx b/resources/ResourceFile_HIV.xlsx
index 00f7b684db..64ef25c261 100644
--- a/resources/ResourceFile_HIV.xlsx
+++ b/resources/ResourceFile_HIV.xlsx
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:b34a88635b02ee8a465462c8eb67a485d721c9159a5bba1df8e63609b803ebe9
-size 161679
+oid sha256:e2345032931c1360046dc7394681cc39669687888f7f8f3e42469d8add067438
+size 160376
diff --git a/resources/ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx b/resources/ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx
index 1586c251f4..ff88584e3f 100644
--- a/resources/ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx
+++ b/resources/ResourceFile_Improved_Healthsystem_And_Healthcare_Seeking.xlsx
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:e63c16cbd0a069d9d10cf3c7212c8804fb1a047397227485adf348728fa5403b
-size 48334
+oid sha256:da900375dda86f999e744bfb6d6dec7347d5b13f176046ba182740421a43d256
+size 48274
diff --git a/resources/ResourceFile_RTI.xlsx b/resources/ResourceFile_RTI.xlsx
index 553d6febb0..68cdd18422 100644
--- a/resources/ResourceFile_RTI.xlsx
+++ b/resources/ResourceFile_RTI.xlsx
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:d950c5d769848fb226db8c1a7d7796c8e43cc2590806f846b98a4bbef6840948
-size 13776
+oid sha256:2c11ada2e8b77675950b61fc8e0efd1c4fa35dffaecaf1029eafd61892a7cefb
+size 13949
diff --git a/resources/ResourceFile_TB.xlsx b/resources/ResourceFile_TB.xlsx
index 2b612ad6ec..d40eb40490 100644
--- a/resources/ResourceFile_TB.xlsx
+++ b/resources/ResourceFile_TB.xlsx
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:93d7bf76c8bece548e08e3f0cb6e9e28a09ca2b5760a408399bf9641f7ed2001
-size 56523
+oid sha256:3fc295cc70b8e86c75e1725a92ceb95bc86a26d3fbe1f680db379726bcab3ab3
+size 55662
diff --git a/resources/costing/ResourceFile_Costing.xlsx b/resources/costing/ResourceFile_Costing.xlsx
index 9f06132aaa..cc427e145c 100644
--- a/resources/costing/ResourceFile_Costing.xlsx
+++ b/resources/costing/ResourceFile_Costing.xlsx
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:2afa3649672e10c9741b26dc70fa0f4496af4fccfafbf8b7b70f3b90b291a4fb
-size 1007463
+oid sha256:224bf212ab783a5cd673d27a588345cb78062187cad0566c06dc6b2fe04dffcd
+size 4307162
diff --git a/resources/epilepsy/ResourceFile_Epilepsy.xlsx b/resources/epilepsy/ResourceFile_Epilepsy.xlsx
index 8bfa24affb..4bdf5ee91c 100644
--- a/resources/epilepsy/ResourceFile_Epilepsy.xlsx
+++ b/resources/epilepsy/ResourceFile_Epilepsy.xlsx
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:94938f9187d5573f068f458263cb6d37ca3ce776eb8dfc9542e5cee0543c8804
-size 1250009
+oid sha256:3e3c38418df28aabb98602e1b00e77d3840143a9fff8de495230817042d2ed45
+size 1250058
diff --git a/resources/healthsystem/ResourceFile_HealthSystem_parameters.csv b/resources/healthsystem/ResourceFile_HealthSystem_parameters.csv
index 8c670a914d..6ca37170a7 100644
--- a/resources/healthsystem/ResourceFile_HealthSystem_parameters.csv
+++ b/resources/healthsystem/ResourceFile_HealthSystem_parameters.csv
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:f1cb38ba76c5673855e2e17e28ad1d36b5cec07d5d7872a7d6bc8aafde0f7009
-size 828
+oid sha256:5df67165565fc88987f848e43363616e2ea4135de7c74d131e785ddc0178f123
+size 706
diff --git a/resources/healthsystem/consumables/ResourceFile_Consumables_availability_and_usage.csv b/resources/healthsystem/consumables/ResourceFile_Consumables_availability_and_usage.csv
new file mode 100644
index 0000000000..4d7f1cb90e
--- /dev/null
+++ b/resources/healthsystem/consumables/ResourceFile_Consumables_availability_and_usage.csv
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b91ddbc76e9833e4b1262d187988dd568be63a8d448785986577d40164c8f02b
+size 115910333
diff --git a/resources/healthsystem/consumables/ResourceFile_Consumables_availability_small.csv b/resources/healthsystem/consumables/ResourceFile_Consumables_availability_small.csv
index 25249531b2..54453cbc2f 100644
--- a/resources/healthsystem/consumables/ResourceFile_Consumables_availability_small.csv
+++ b/resources/healthsystem/consumables/ResourceFile_Consumables_availability_small.csv
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:c358a643e4def0e574b75f89f83d77f9c3366f668422e005150f4d69ebe8d7a7
-size 6169152
+oid sha256:69a5143c0b7307c7bb48726aa73d6c2f61de2a69aeb445eec87494cf9d4a1041
+size 6087331
diff --git a/resources/healthsystem/consumables/ResourceFile_consumables_matched.csv b/resources/healthsystem/consumables/ResourceFile_consumables_matched.csv
index 7ab675ecba..980adbfba2 100644
--- a/resources/healthsystem/consumables/ResourceFile_consumables_matched.csv
+++ b/resources/healthsystem/consumables/ResourceFile_consumables_matched.csv
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:b5b0f417681cbdd2489e2f9c6634b2825c32beb9637dc045b56e308c910a102c
-size 90569
+oid sha256:084aa9b6e011321538e907e0860a57b6392e9b16fd848bfb982374365e5fa286
+size 90079
diff --git a/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_HR_scaling_by_level_and_officer_type.xlsx b/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_HR_scaling_by_level_and_officer_type.xlsx
index e7f34296e6..3d804bbc77 100644
--- a/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_HR_scaling_by_level_and_officer_type.xlsx
+++ b/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_HR_scaling_by_level_and_officer_type.xlsx
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:5f6e1a0c8ec505dd613dfc9c0b1b14d16ee3161500bc08c743398754d2074203
-size 15682
+oid sha256:af86c2c2af5c291c18c5d481681d6d316526b81806c8c8e898517e850160e6fd
+size 12465
diff --git a/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_dynamic_HR_scaling.xlsx b/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_dynamic_HR_scaling.xlsx
index a633e6fc92..36b9fd0dc2 100644
--- a/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_dynamic_HR_scaling.xlsx
+++ b/resources/healthsystem/human_resources/scaling_capabilities/ResourceFile_dynamic_HR_scaling.xlsx
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:d2d74390498e497ee0bf68773327868f6b199c1c9569337b173fa330c0f2f926
-size 24593
+oid sha256:b8388ef18f073c9470c01f8408bff572017484763cfc4c87bb0212c38ee0b6d7
+size 25488
diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv
index 45f801f0c8..33ba052c64 100644
--- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv
+++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_EquipmentCatalogue.csv
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:ec5f619816df6150ae92839152607296a5f2289024c92ce6b5ba621d38db20b7
-size 33517
+oid sha256:3e151e16f7eea2ae61d2fa637c26449aa533ddc6a7f0d83aff495f5f6c9d1f8d
+size 33201
diff --git a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment_Availability_Estimates.csv b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment_Availability_Estimates.csv
index 706297da67..3f0739577a 100644
--- a/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment_Availability_Estimates.csv
+++ b/resources/healthsystem/infrastructure_and_equipment/ResourceFile_Equipment_Availability_Estimates.csv
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:2785365d20a4da4c147ba6a5df9e0259c9076df0fec556086aea0f2a068c9c53
-size 1313098
+oid sha256:e31936377f6b90779ad66480c4bc477cdca9322e86f2e00d202bbb91eebf6d57
+size 1306170
diff --git a/resources/healthsystem/organisation/ResourceFile_Master_Facilities_List.csv b/resources/healthsystem/organisation/ResourceFile_Master_Facilities_List.csv
index 9468bcf080..5ebedf3aab 100644
--- a/resources/healthsystem/organisation/ResourceFile_Master_Facilities_List.csv
+++ b/resources/healthsystem/organisation/ResourceFile_Master_Facilities_List.csv
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:73bfb5a34b54939cbaf842feceef1013c6309d3b1e0a5ef27ffc84577ac3519e
-size 6602
+oid sha256:c6df4a42409b22d0b10d56ec077f6f4b5ccbed0f16f570fedbfd397e100063a9
+size 8471
diff --git a/resources/malaria/ResourceFile_malaria.xlsx b/resources/malaria/ResourceFile_malaria.xlsx
index 7537f3ace9..70902b7480 100644
--- a/resources/malaria/ResourceFile_malaria.xlsx
+++ b/resources/malaria/ResourceFile_malaria.xlsx
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
-oid sha256:7f256d5007b36e2428ae844747bd766bb6086540c5135408d606dd821e185d9f
-size 69578
+oid sha256:6ba5849e265103ee799d1982325b6fed1ef4d3df559ffce9d6790395c201fcaf
+size 67562
diff --git a/src/scripts/automation/mark_slow_tests.py b/src/scripts/automation/mark_slow_tests.py
deleted file mode 100644
index daa507e2ab..0000000000
--- a/src/scripts/automation/mark_slow_tests.py
+++ /dev/null
@@ -1,298 +0,0 @@
-"""Script to automatically mark slow running tests with `pytest.mark.slow` decorator."""
-
-
-import argparse
-import difflib
-import json
-import re
-import warnings
-from collections import defaultdict
-from pathlib import Path
-from typing import Dict, NamedTuple, Optional, Set, Tuple, Union
-
-import redbaron
-
-SLOW_MARK_DECORATOR = "pytest.mark.slow"
-
-
-class TestFunction(NamedTuple):
- module_path: Path
- name: str
-
-
-class TestMethod(NamedTuple):
- module_path: Path
- class_name: str
- method_name: str
-
-
-TestNode = Union[TestFunction, TestMethod]
-
-
-def parse_nodeid_last_part(last_part: str) -> Tuple[str, Optional[str]]:
- match = re.match(r"(.+)\[(.+)\]", last_part)
- if match is not None:
- return match[1], match[2]
- else:
- return last_part, None
-
-
-def parse_nodeid(nodeid: str) -> TestNode:
- parts = nodeid.split("::")
- if len(parts) == 2:
- module_path, last_part = parts
- name, _ = parse_nodeid_last_part(last_part)
- return TestFunction(Path(module_path), name)
- elif len(parts) == 3:
- module_path, class_name, last_part = parts
- method_name, _ = parse_nodeid_last_part(last_part)
- return TestMethod(Path(module_path), class_name, method_name)
- else:
- msg = f"Test nodeid has unexpected format: {nodeid}"
- raise ValueError(msg)
-
-
-def parse_test_report(
- json_test_report_path: Path,
- remove_slow_threshold: float,
- add_slow_threshold: float,
-) -> Dict[Path, Dict[str, Set[TestNode]]]:
- with open(json_test_report_path, "r") as f:
- test_report = json.load(f)
- tests_to_change_slow_mark_by_module: defaultdict = defaultdict(
- lambda: {"add": set(), "remove": set()}
- )
- tests_to_keep_slow_mark_by_module: defaultdict = defaultdict(set)
- for test in test_report["tests"]:
- if test["outcome"] != "passed":
- continue
- test_node = parse_nodeid(test["nodeid"])
- marked_slow = "slow" in test["keywords"]
- call_duration = test["call"]["duration"]
- if marked_slow and call_duration < remove_slow_threshold:
- tests_to_change_slow_mark_by_module[test_node.module_path]["remove"].add(
- test_node
- )
- elif not marked_slow and call_duration > add_slow_threshold:
- tests_to_change_slow_mark_by_module[test_node.module_path]["add"].add(
- test_node
- )
- elif marked_slow:
- tests_to_keep_slow_mark_by_module[test_node.module_path].add(test_node)
- # Parameterized tests may have different call durations for different parameters
- # however slow mark applies to all parameters, therefore if any tests appear in
- # both set of tests to keep slow mark and test to remove slow mark (corresponding
- # to runs of same test with different parameters) we remove them from the set of
- # tests to remove slow mark
- for (
- module_path,
- test_nodes_to_change,
- ) in tests_to_change_slow_mark_by_module.items():
- test_nodes_to_change["remove"].difference_update(
- tests_to_keep_slow_mark_by_module[module_path]
- )
- return dict(tests_to_change_slow_mark_by_module)
-
-
-def find_function(
- module_fst: redbaron.RedBaron, function_name: str
-) -> redbaron.DefNode:
- return module_fst.find("def", lambda node: node.name == function_name)
-
-
-def find_class_method(
- module_fst: redbaron.RedBaron, class_name: str, method_name: str
-) -> redbaron.DefNode:
- class_fst = module_fst.find("class", lambda node: node.name == class_name)
- return class_fst.fund("def", lambda node: node.name == method_name)
-
-
-def find_decorator(
- function_fst: redbaron.DefNode, decorator_code: str
-) -> redbaron.DecoratorNode:
- return function_fst.find(
- "decorator", lambda node: str(node.value) == decorator_code
- )
-
-
-def add_decorator(function_fst: redbaron.DefNode, decorator_code: str):
- if len(function_fst.decorators) == 0:
- function_fst.decorators = f"@{decorator_code}"
- else:
- function_fst.decorators.append(f"@{decorator_code}")
-
-
-def remove_decorator(
- function_fst: redbaron.DefNode, decorator_fst: redbaron.DecoratorNode
-):
- # Need to remove both decorator and associated end line node so we find index of
- # decorator and pop it and next node (which should be end line node) rather than
- # use remove method of decorators proxy list directly
- decorator_index = function_fst.decorators.node_list.index(decorator_fst)
- popped_decorator_fst = function_fst.decorators.node_list.pop(decorator_index)
- endline_fst = function_fst.decorators.node_list.pop(decorator_index)
- if popped_decorator_fst is not decorator_fst or not isinstance(
- endline_fst, redbaron.EndlNode
- ):
- msg = (
- f"Removed {popped_decorator_fst} and {endline_fst} when expecting "
- f"{decorator_fst} and end line node."
- )
- raise RuntimeError(msg)
-
-
-def remove_mark_from_tests(
- module_fst: redbaron.RedBaron,
- tests_to_remove_mark: Set[TestNode],
- mark_decorator: str,
-):
- for test_node in tests_to_remove_mark:
- if isinstance(test_node, TestFunction):
- function_fst = find_function(module_fst, test_node.name)
- else:
- function_fst = find_class_method(
- module_fst, test_node.class_name, test_node.method_name
- )
- decorator_fst = find_decorator(function_fst, mark_decorator)
- if decorator_fst is None:
- msg = (
- f"Test {test_node} unexpectedly does not have a decorator "
- f"{mark_decorator} - this suggests you may be using a JSON test report "
- "generated using a different version of tests code."
- )
- warnings.warn(msg, stacklevel=2)
- else:
- remove_decorator(function_fst, decorator_fst)
-
-
-def add_mark_to_tests(
- module_fst: redbaron.RedBaron, tests_to_add_mark: Set[TestNode], mark_decorator: str
-):
- for test_node in tests_to_add_mark:
- if isinstance(test_node, TestFunction):
- function_fst = find_function(module_fst, test_node.name)
- else:
- function_fst = find_class_method(
- module_fst, test_node.class_name, test_node.method_name
- )
- if find_decorator(function_fst, mark_decorator) is not None:
- msg = (
- f"Test {test_node} unexpectedly already has a decorator "
- f"{mark_decorator} - this suggests you may be using a JSON test report "
- "generated using a different version of tests code."
- )
- warnings.warn(msg, stacklevel=2)
- else:
- add_decorator(function_fst, mark_decorator)
-
-
-def add_import(module_fst: redbaron.RedBaron, module_name: str):
- last_top_level_import = module_fst.find_all(
- "import", lambda node: node.parent is module_fst
- )[-1]
- import_statement = f"import {module_name}"
- if last_top_level_import is not None:
- last_top_level_import.insert_after(import_statement)
- else:
- if isinstance(module_fst[0], redbaron.Nodes.StringNode):
- module_fst[0].insert_after(import_statement)
- else:
- module_fst[0].insert_before(import_statement)
-
-
-def remove_import(module_fst: redbaron.RedBaron, module_name: str):
- import_fst = module_fst.find("import", lambda node: module_name in node.modules())
- if len(import_fst.modules()) > 1:
- import_fst.remove(module_name)
- else:
- module_fst.remove(import_fst)
-
-
-def update_test_slow_marks(
- tests_to_change_slow_mark_by_module: Dict[Path, Dict[str, Set[TestNode]]],
- show_diff: bool,
-):
- for (
- module_path,
- test_nodes_to_change,
- ) in tests_to_change_slow_mark_by_module.items():
- with open(module_path, "r") as source_code:
- module_fst = redbaron.RedBaron(source_code.read())
- original_module_fst = module_fst.copy()
- remove_mark_from_tests(
- module_fst, test_nodes_to_change["remove"], SLOW_MARK_DECORATOR
- )
- add_mark_to_tests(module_fst, test_nodes_to_change["add"], SLOW_MARK_DECORATOR)
- any_marked = (
- module_fst.find(
- "decorator", lambda node: str(node.value) == SLOW_MARK_DECORATOR
- )
- is not None
- )
- pytest_imported = (
- module_fst.find("import", lambda node: "pytest" in node.modules())
- is not None
- )
- if any_marked and not pytest_imported:
- add_import(module_fst, "pytest")
- elif not any_marked and pytest_imported:
- pytest_references = module_fst.find_all("name", "pytest")
- if (
- len(pytest_references) == 1
- and pytest_references[0].parent_find("import") is not None
- ):
- remove_import(module_fst, "pytest")
- if show_diff:
- diff_lines = difflib.unified_diff(
- original_module_fst.dumps().split("\n"),
- module_fst.dumps().split("\n"),
- fromfile=str(module_path),
- tofile=f"Updated {module_path}",
- )
- print("\n".join(diff_lines), end="")
- else:
- with open(module_path, "w") as source_code:
- source_code.write(module_fst.dumps())
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser("Mark slow running tests with pytest.mark.slow")
- parser.add_argument(
- "--json-test-report-path",
- type=Path,
- help="JSON report output from pytest-json-report plugin listing test durations",
- )
- parser.add_argument(
- "--remove-slow-threshold",
- type=float,
- default=9.0,
- help="Threshold in seconds for test duration below which to remove slow marker",
- )
- parser.add_argument(
- "--add-slow-threshold",
- type=float,
- default=11.0,
- help="Threshold in seconds for test duration above which to add slow marker",
- )
- parser.add_argument(
- "--show-diff",
- action="store_true",
- help="Print line-by-line diff of changes to stdout without changing files",
- )
- args = parser.parse_args()
- if not args.json_test_report_path.exists():
- msg = f"No file found at --json-test-report-path={args.json_test_report_path}"
- raise FileNotFoundError(msg)
- # We want a hysteresis effect by having remove_slow_threshold < add_slow_threshold
- # so a test with duration close to the thresholds doesn't keep getting marks added
- # and removed due to noise in durations
- if args.remove_slow_threshold > args.add_slow_threshold:
- msg = (
- "Argument --remove-slow-threshold should be less than or equal to "
- "--add-slow-threshold"
- )
- raise ValueError(msg)
- tests_to_change_slow_mark_by_module = parse_test_report(
- args.json_test_report_path, args.remove_slow_threshold, args.add_slow_threshold
- )
- update_test_slow_marks(tests_to_change_slow_mark_by_module, args.show_diff)
diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_hss_elements.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_hss_elements.py
deleted file mode 100644
index 76708f7c25..0000000000
--- a/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_hss_elements.py
+++ /dev/null
@@ -1,272 +0,0 @@
-"""Produce plots to show the impact each the healthcare system (overall health impact) when running under different
-scenarios (scenario_impact_of_healthsystem.py)"""
-
-import argparse
-import textwrap
-from pathlib import Path
-from typing import Tuple
-
-import numpy as np
-import pandas as pd
-from matplotlib import pyplot as plt
-
-from tlo import Date
-from tlo.analysis.utils import extract_results, make_age_grp_lookup, summarize
-
-
-def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = None):
- """Produce standard set of plots describing the effect of each TREATMENT_ID.
- - We estimate the epidemiological impact as the EXTRA deaths that would occur if that treatment did not occur.
- - We estimate the draw on healthcare system resources as the FEWER appointments when that treatment does not occur.
- """
-
- TARGET_PERIOD = (Date(2020, 1, 1), Date(2030, 12, 31))
-
- # Definitions of general helper functions
- make_graph_file_name = lambda stub: output_folder / f"{stub.replace('*', '_star_')}.png" # noqa: E731
-
- _, age_grp_lookup = make_age_grp_lookup()
-
- def target_period() -> str:
- """Returns the target period as a string of the form YYYY-YYYY"""
- return "-".join(str(t.year) for t in TARGET_PERIOD)
-
- def get_parameter_names_from_scenario_file() -> Tuple[str]:
- """Get the tuple of names of the scenarios from `Scenario` class used to create the results."""
- from scripts.comparison_of_horizontal_and_vertical_programs.scenario_hss_elements import (
- HSSElements,
- )
- e = HSSElements()
- return tuple(e._scenarios.keys())
-
- def get_num_deaths(_df):
- """Return total number of Deaths (total within the TARGET_PERIOD)"""
- return pd.Series(data=len(_df.loc[pd.to_datetime(_df.date).between(*TARGET_PERIOD)]))
-
- def get_num_dalys(_df):
- """Return total number of DALYS (Stacked) by label (total within the TARGET_PERIOD).
- Throw error if not a record for every year in the TARGET PERIOD (to guard against inadvertently using
- results from runs that crashed mid-way through the simulation.
- """
- years_needed = [i.year for i in TARGET_PERIOD]
- assert set(_df.year.unique()).issuperset(years_needed), "Some years are not recorded."
- return pd.Series(
- data=_df
- .loc[_df.year.between(*years_needed)]
- .drop(columns=['date', 'sex', 'age_range', 'year'])
- .sum().sum()
- )
-
- def set_param_names_as_column_index_level_0(_df):
- """Set the columns index (level 0) as the param_names."""
- ordered_param_names_no_prefix = {i: x for i, x in enumerate(param_names)}
- names_of_cols_level0 = [ordered_param_names_no_prefix.get(col) for col in _df.columns.levels[0]]
- assert len(names_of_cols_level0) == len(_df.columns.levels[0])
- _df.columns = _df.columns.set_levels(names_of_cols_level0, level=0)
- return _df
-
- def find_difference_relative_to_comparison(_ser: pd.Series,
- comparison: str,
- scaled: bool = False,
- drop_comparison: bool = True,
- ):
- """Find the difference in the values in a pd.Series with a multi-index, between the draws (level 0)
- within the runs (level 1), relative to where draw = `comparison`.
- The comparison is `X - COMPARISON`."""
- return _ser \
- .unstack(level=0) \
- .apply(lambda x: (x - x[comparison]) / (x[comparison] if scaled else 1.0), axis=1) \
- .drop(columns=([comparison] if drop_comparison else [])) \
- .stack()
-
- def do_bar_plot_with_ci(_df, annotations=None, xticklabels_horizontal_and_wrapped=False, put_labels_in_legend=True):
- """Make a vertical bar plot for each row of _df, using the columns to identify the height of the bar and the
- extent of the error bar."""
-
- substitute_labels = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
-
- yerr = np.array([
- (_df['mean'] - _df['lower']).values,
- (_df['upper'] - _df['mean']).values,
- ])
-
- xticks = {(i + 0.5): k for i, k in enumerate(_df.index)}
-
- # Define colormap (used only with option `put_labels_in_legend=True`)
- cmap = plt.get_cmap("tab20")
- rescale = lambda y: (y - np.min(y)) / (np.max(y) - np.min(y)) # noqa: E731
- colors = list(map(cmap, rescale(np.array(list(xticks.keys()))))) if put_labels_in_legend else None
-
- fig, ax = plt.subplots(figsize=(10, 5))
- ax.bar(
- xticks.keys(),
- _df['mean'].values,
- yerr=yerr,
- alpha=0.8,
- ecolor='black',
- color=colors,
- capsize=10,
- label=xticks.values()
- )
- if annotations:
- for xpos, ypos, text in zip(xticks.keys(), _df['upper'].values, annotations):
- ax.text(xpos, ypos*1.15, text, horizontalalignment='center', rotation='vertical', fontsize='x-small')
- ax.set_xticks(list(xticks.keys()))
-
- if put_labels_in_legend:
- # Update xticks label with substitute labels
- # Insert legend with updated labels that shows correspondence between substitute label and original label
- xtick_values = [letter for letter, label in zip(substitute_labels, xticks.values())]
- xtick_legend = [f'{letter}: {label}' for letter, label in zip(substitute_labels, xticks.values())]
- h, legs = ax.get_legend_handles_labels()
- ax.legend(h, xtick_legend, loc='center left', fontsize='small', bbox_to_anchor=(1, 0.5))
- ax.set_xticklabels(list(xtick_values))
- else:
- if not xticklabels_horizontal_and_wrapped:
- # xticklabels will be vertical and not wrapped
- ax.set_xticklabels(list(xticks.values()), rotation=90)
- else:
- wrapped_labs = ["\n".join(textwrap.wrap(_lab, 20)) for _lab in xticks.values()]
- ax.set_xticklabels(wrapped_labs)
-
- ax.grid(axis="y")
- ax.spines['top'].set_visible(False)
- ax.spines['right'].set_visible(False)
- fig.tight_layout()
-
- return fig, ax
-
- # %% Define parameter names
- param_names = get_parameter_names_from_scenario_file()
-
- # %% Quantify the health gains associated with all interventions combined.
-
- # Absolute Number of Deaths and DALYs
- num_deaths = extract_results(
- results_folder,
- module='tlo.methods.demography',
- key='death',
- custom_generate_series=get_num_deaths,
- do_scaling=True
- ).pipe(set_param_names_as_column_index_level_0)
-
- num_dalys = extract_results(
- results_folder,
- module='tlo.methods.healthburden',
- key='dalys_stacked',
- custom_generate_series=get_num_dalys,
- do_scaling=True
- ).pipe(set_param_names_as_column_index_level_0)
-
- # %% Charts of total numbers of deaths / DALYS
- num_dalys_summarized = summarize(num_dalys).loc[0].unstack().reindex(param_names)
- num_deaths_summarized = summarize(num_deaths).loc[0].unstack().reindex(param_names)
-
- name_of_plot = f'Deaths, {target_period()}'
- fig, ax = do_bar_plot_with_ci(num_deaths_summarized / 1e6)
- ax.set_title(name_of_plot)
- ax.set_ylabel('(Millions)')
- fig.tight_layout()
- ax.axhline(num_deaths_summarized.loc['Baseline', 'mean']/1e6, color='black', alpha=0.5)
- fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', '')))
- fig.show()
- plt.close(fig)
-
- name_of_plot = f'All Scenarios: DALYs, {target_period()}'
- fig, ax = do_bar_plot_with_ci(num_dalys_summarized / 1e6)
- ax.set_title(name_of_plot)
- ax.set_ylabel('(Millions)')
- ax.axhline(num_dalys_summarized.loc['Baseline', 'mean']/1e6, color='black', alpha=0.5)
- fig.tight_layout()
- fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', '')))
- fig.show()
- plt.close(fig)
-
-
- # %% Deaths and DALYS averted relative to Status Quo
- num_deaths_averted = summarize(
- -1.0 *
- pd.DataFrame(
- find_difference_relative_to_comparison(
- num_deaths.loc[0],
- comparison='Baseline')
- ).T
- ).iloc[0].unstack().reindex(param_names).drop(['Baseline'])
-
- pc_deaths_averted = 100.0 * summarize(
- -1.0 *
- pd.DataFrame(
- find_difference_relative_to_comparison(
- num_deaths.loc[0],
- comparison='Baseline',
- scaled=True)
- ).T
- ).iloc[0].unstack().reindex(param_names).drop(['Baseline'])
-
- num_dalys_averted = summarize(
- -1.0 *
- pd.DataFrame(
- find_difference_relative_to_comparison(
- num_dalys.loc[0],
- comparison='Baseline')
- ).T
- ).iloc[0].unstack().reindex(param_names).drop(['Baseline'])
-
- pc_dalys_averted = 100.0 * summarize(
- -1.0 *
- pd.DataFrame(
- find_difference_relative_to_comparison(
- num_dalys.loc[0],
- comparison='Baseline',
- scaled=True)
- ).T
- ).iloc[0].unstack().reindex(param_names).drop(['Baseline'])
-
- # DEATHS
- name_of_plot = f'Additional Deaths Averted vs Baseline, {target_period()}'
- fig, ax = do_bar_plot_with_ci(
- num_deaths_averted.clip(lower=0.0),
- annotations=[
- f"{round(row['mean'], 0)} ({round(row['lower'], 1)}-{round(row['upper'], 1)}) %"
- for _, row in pc_deaths_averted.clip(lower=0.0).iterrows()
- ]
- )
- ax.set_title(name_of_plot)
- ax.set_ylabel('Additional Deaths Averted')
- fig.tight_layout()
- fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', '')))
- fig.show()
- plt.close(fig)
-
- # DALYS
- name_of_plot = f'Additional DALYs Averted vs Baseline, {target_period()}'
- fig, ax = do_bar_plot_with_ci(
- (num_dalys_averted / 1e6).clip(lower=0.0),
- annotations=[
- f"{round(row['mean'])} ({round(row['lower'], 1)}-{round(row['upper'], 1)}) %"
- for _, row in pc_dalys_averted.clip(lower=0.0).iterrows()
- ]
- )
- ax.set_title(name_of_plot)
- ax.set_ylabel('Additional DALYS Averted \n(Millions)')
- fig.tight_layout()
- fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', '')))
- fig.show()
- plt.close(fig)
-
- # todo: Neaten graphs
- # todo: Graph showing difference broken down by disease (this can be cribbed from the calcs about wealth from the
- # third set of analyses in the overview paper).
- # todo: other metrics of health
- # todo: other graphs, broken down by age/sex (this can also be cribbed from overview paper stuff)
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser()
- parser.add_argument("results_folder", type=Path) # outputs/horizontal_and_vertical_programs-2024-05-16
- args = parser.parse_args()
-
- apply(
- results_folder=args.results_folder,
- output_folder=args.results_folder,
- resourcefilepath=Path('./resources')
- )
diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py
deleted file mode 100644
index 0cfcd05315..0000000000
--- a/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py
+++ /dev/null
@@ -1,229 +0,0 @@
-"""
-This scenario file sets up the scenarios for simulating the effects of scaling up programs
-
-The scenarios are:
-*0 baseline mode 1
-*1 scale-up HIV program
-*2 scale-up TB program
-*3 scale-up malaria program
-*4 scale-up HIV and Tb and malaria programs
-
-scale-up occurs on the default scale-up start date (01/01/2025: in parameters list of resourcefiles)
-
-For all scenarios, keep all default health system settings
-
-check the batch configuration gets generated without error:
-tlo scenario-run --draw-only src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py
-
-Run on the batch system using:
-tlo batch-submit src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py
-
-or locally using:
-tlo scenario-run src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py
-
-or execute a single run:
-tlo scenario-run src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_maxHTM_scenario.py --draw 1 0
-
-"""
-
-import datetime
-from pathlib import Path
-
-from tlo import Date, logging
-from tlo.methods import (
- demography,
- enhanced_lifestyle,
- epi,
- healthburden,
- healthseekingbehaviour,
- healthsystem,
- hiv,
- malaria,
- simplified_births,
- symptommanager,
- tb,
-)
-from tlo.scenario import BaseScenario
-
-resourcefilepath = Path("./resources")
-datestamp = datetime.date.today().strftime("__%Y_%m_%d")
-
-outputspath = Path("./outputs")
-scaleup_start_year = 2012
-end_date = Date(2015, 1, 1)
-
-
-class EffectOfProgrammes(BaseScenario):
- def __init__(self):
- super().__init__()
- self.seed = 0
- self.start_date = Date(2010, 1, 1)
- self.end_date = end_date
- self.pop_size = 1_000
- self.number_of_draws = 5
- self.runs_per_draw = 1
-
- def log_configuration(self):
- return {
- 'filename': 'scaleup_tests',
- 'directory': Path('./outputs'), # <- (specified only for local running)
- 'custom_levels': {
- '*': logging.WARNING,
- 'tlo.methods.hiv': logging.INFO,
- 'tlo.methods.tb': logging.INFO,
- 'tlo.methods.malaria': logging.INFO,
- 'tlo.methods.demography': logging.INFO,
- }
- }
-
- def modules(self):
- return [
- demography.Demography(resourcefilepath=self.resources),
- simplified_births.SimplifiedBirths(resourcefilepath=self.resources),
- enhanced_lifestyle.Lifestyle(resourcefilepath=self.resources),
- healthsystem.HealthSystem(resourcefilepath=self.resources),
- symptommanager.SymptomManager(resourcefilepath=self.resources),
- healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=self.resources),
- healthburden.HealthBurden(resourcefilepath=self.resources),
- epi.Epi(resourcefilepath=self.resources),
- hiv.Hiv(resourcefilepath=self.resources),
- tb.Tb(resourcefilepath=self.resources),
- malaria.Malaria(resourcefilepath=self.resources),
- ]
-
- def draw_parameters(self, draw_number, rng):
-
- return {
- 'Hiv': {
- 'type_of_scaleup': ['none', 'max', 'none', 'none', 'max'][draw_number],
- 'scaleup_start_year': scaleup_start_year,
- },
- 'Tb': {
- 'type_of_scaleup': ['none', 'none', 'max', 'none', 'max'][draw_number],
- 'scaleup_start_year': scaleup_start_year,
- },
- 'Malaria': {
- 'type_of_scaleup': ['none', 'none', 'none', 'max', 'max'][draw_number],
- 'scaleup_start_year': scaleup_start_year,
- },
- }
-
-
-if __name__ == '__main__':
- from tlo.cli import scenario_run
-
- scenario_run([__file__])
-
-
-
-# %% Produce some figures and summary info
-
-# import pandas as pd
-# import matplotlib.pyplot as plt
-
-# # Find results_folder associated with a given batch_file (and get most recent [-1])
-# results_folder = get_scenario_outputs("scaleup_tests-", outputspath)[-1]
-#
-# # get basic information about the results
-# info = get_scenario_info(results_folder)
-#
-# # 1) Extract the parameters that have varied over the set of simulations
-# params = extract_params(results_folder)
-#
-#
-# # DEATHS
-#
-#
-# def get_num_deaths_by_cause_label(_df):
-# """Return total number of Deaths by label within the TARGET_PERIOD
-# values are summed for all ages
-# df returned: rows=COD, columns=draw
-# """
-# return _df \
-# .loc[pd.to_datetime(_df.date).between(*TARGET_PERIOD)] \
-# .groupby(_df['label']) \
-# .size()
-#
-#
-# TARGET_PERIOD = (Date(scaleup_start_year, 1, 1), end_date)
-#
-# # produce df of total deaths over scale-up period
-# num_deaths_by_cause_label = extract_results(
-# results_folder,
-# module='tlo.methods.demography',
-# key='death',
-# custom_generate_series=get_num_deaths_by_cause_label,
-# do_scaling=True
-# )
-#
-#
-# def summarise_deaths_for_one_cause(results_folder, label):
-# """ returns mean deaths for each year of the simulation
-# values are aggregated across the runs of each draw
-# for the specified cause
-# """
-#
-# results_deaths = extract_results(
-# results_folder,
-# module="tlo.methods.demography",
-# key="death",
-# custom_generate_series=(
-# lambda df: df.assign(year=df["date"].dt.year).groupby(
-# ["year", "label"])["person_id"].count()
-# ),
-# do_scaling=True,
-# )
-# # removes multi-index
-# results_deaths = results_deaths.reset_index()
-#
-# # select only cause specified
-# tmp = results_deaths.loc[
-# (results_deaths.label == label)
-# ]
-#
-# # group deaths by year
-# tmp = pd.DataFrame(tmp.groupby(["year"]).sum())
-#
-# # get mean for each draw
-# mean_deaths = pd.concat({'mean': tmp.iloc[:, 1:].groupby(level=0, axis=1).mean()}, axis=1).swaplevel(axis=1)
-#
-# return mean_deaths
-#
-#
-# aids_deaths = summarise_deaths_for_one_cause(results_folder, 'AIDS')
-# tb_deaths = summarise_deaths_for_one_cause(results_folder, 'TB (non-AIDS)')
-# malaria_deaths = summarise_deaths_for_one_cause(results_folder, 'Malaria')
-#
-#
-# draw_labels = ['No scale-up', 'HIV scale-up', 'TB scale-up', 'Malaria scale-up', 'HTM scale-up']
-# colours = ['blue', 'green', 'red', 'purple', 'orange']
-#
-# # Create subplots
-# fig, axs = plt.subplots(3, 1, figsize=(10, 10))
-# # Plot for df1
-# for i, col in enumerate(aids_deaths.columns):
-# axs[0].plot(aids_deaths.index, aids_deaths[col], label=draw_labels[i],
-# color=colours[i])
-# axs[0].set_title('HIV/AIDS')
-# axs[0].legend(loc='center left', bbox_to_anchor=(1, 0.5)) # Legend to the right of the plot
-# axs[0].axvline(x=scaleup_start_year, color='gray', linestyle='--')
-#
-# # Plot for df2
-# for i, col in enumerate(tb_deaths.columns):
-# axs[1].plot(tb_deaths.index, tb_deaths[col], color=colours[i])
-# axs[1].set_title('TB')
-# axs[1].axvline(x=scaleup_start_year, color='gray', linestyle='--')
-#
-# # Plot for df3
-# for i, col in enumerate(malaria_deaths.columns):
-# axs[2].plot(malaria_deaths.index, malaria_deaths[col], color=colours[i])
-# axs[2].set_title('Malaria')
-# axs[2].axvline(x=scaleup_start_year, color='gray', linestyle='--')
-#
-# for ax in axs:
-# ax.set_xlabel('Years')
-# ax.set_ylabel('Number deaths')
-#
-# plt.tight_layout(rect=[0, 0, 0.85, 1]) # Adjust layout to make space for legend
-# plt.show()
-#
diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_vertical_programs_with_and_without_hss.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_vertical_programs_with_and_without_hss.py
deleted file mode 100644
index f0dd083d97..0000000000
--- a/src/scripts/comparison_of_horizontal_and_vertical_programs/analysis_vertical_programs_with_and_without_hss.py
+++ /dev/null
@@ -1,363 +0,0 @@
-"""Produce plots to show the impact each the healthcare system (overall health impact) when running under different
-scenarios (scenario_impact_of_healthsystem.py)"""
-
-import argparse
-import textwrap
-from pathlib import Path
-from typing import Tuple
-
-import numpy as np
-import pandas as pd
-from matplotlib import pyplot as plt
-
-from tlo import Date
-from tlo.analysis.utils import extract_results, make_age_grp_lookup, summarize
-
-
-def apply(results_folder: Path, output_folder: Path, resourcefilepath: Path = None):
- """Produce standard set of plots describing the effect of each TREATMENT_ID.
- - We estimate the epidemiological impact as the EXTRA deaths that would occur if that treatment did not occur.
- - We estimate the draw on healthcare system resources as the FEWER appointments when that treatment does not occur.
- """
-
- TARGET_PERIOD = (Date(2020, 1, 1), Date(2030, 12, 31))
-
- # Definitions of general helper functions
- make_graph_file_name = lambda stub: output_folder / f"{stub.replace('*', '_star_')}.png" # noqa: E731
-
- _, age_grp_lookup = make_age_grp_lookup()
-
- def target_period() -> str:
- """Returns the target period as a string of the form YYYY-YYYY"""
- return "-".join(str(t.year) for t in TARGET_PERIOD)
-
- def get_parameter_names_from_scenario_file() -> Tuple[str]:
- """Get the tuple of names of the scenarios from `Scenario` class used to create the results."""
- from scripts.comparison_of_horizontal_and_vertical_programs.scenario_vertical_programs_with_and_without_hss import (
- HTMWithAndWithoutHSS,
- )
- e = HTMWithAndWithoutHSS()
- return tuple(e._scenarios.keys())
-
- def get_num_deaths(_df):
- """Return total number of Deaths (total within the TARGET_PERIOD)"""
- return pd.Series(data=len(_df.loc[pd.to_datetime(_df.date).between(*TARGET_PERIOD)]))
-
- def get_num_dalys(_df):
- """Return total number of DALYS (Stacked) by label (total within the TARGET_PERIOD).
- Throw error if not a record for every year in the TARGET PERIOD (to guard against inadvertently using
- results from runs that crashed mid-way through the simulation.
- """
- years_needed = [i.year for i in TARGET_PERIOD]
- assert set(_df.year.unique()).issuperset(years_needed), "Some years are not recorded."
- return pd.Series(
- data=_df
- .loc[_df.year.between(*years_needed)]
- .drop(columns=['date', 'sex', 'age_range', 'year'])
- .sum().sum()
- )
-
- def set_param_names_as_column_index_level_0(_df):
- """Set the columns index (level 0) as the param_names."""
- ordered_param_names_no_prefix = {i: x for i, x in enumerate(param_names)}
- names_of_cols_level0 = [ordered_param_names_no_prefix.get(col) for col in _df.columns.levels[0]]
- assert len(names_of_cols_level0) == len(_df.columns.levels[0])
- _df.columns = _df.columns.set_levels(names_of_cols_level0, level=0)
- return _df
-
- def find_difference_relative_to_comparison_series(
- _ser: pd.Series,
- comparison: str,
- scaled: bool = False,
- drop_comparison: bool = True,
- ):
- """Find the difference in the values in a pd.Series with a multi-index, between the draws (level 0)
- within the runs (level 1), relative to where draw = `comparison`.
- The comparison is `X - COMPARISON`."""
- return _ser \
- .unstack(level=0) \
- .apply(lambda x: (x - x[comparison]) / (x[comparison] if scaled else 1.0), axis=1) \
- .drop(columns=([comparison] if drop_comparison else [])) \
- .stack()
-
- def find_difference_relative_to_comparison_series_dataframe(_df: pd.DataFrame, **kwargs):
- """Apply `find_difference_relative_to_comparison_series` to each row in a dataframe"""
- return pd.concat({
- _idx: find_difference_relative_to_comparison_series(row, **kwargs)
- for _idx, row in _df.iterrows()
- }, axis=1).T
-
- def do_bar_plot_with_ci(_df, annotations=None, xticklabels_horizontal_and_wrapped=False, put_labels_in_legend=True):
- """Make a vertical bar plot for each row of _df, using the columns to identify the height of the bar and the
- extent of the error bar."""
-
- substitute_labels = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
-
- yerr = np.array([
- (_df['mean'] - _df['lower']).values,
- (_df['upper'] - _df['mean']).values,
- ])
-
- xticks = {(i + 0.5): k for i, k in enumerate(_df.index)}
-
- # Define colormap (used only with option `put_labels_in_legend=True`)
- cmap = plt.get_cmap("tab20")
- rescale = lambda y: (y - np.min(y)) / (np.max(y) - np.min(y)) # noqa: E731
- colors = list(map(cmap, rescale(np.array(list(xticks.keys()))))) if put_labels_in_legend else None
-
- fig, ax = plt.subplots(figsize=(10, 5))
- ax.bar(
- xticks.keys(),
- _df['mean'].values,
- yerr=yerr,
- alpha=0.8,
- ecolor='black',
- color=colors,
- capsize=10,
- label=xticks.values()
- )
- if annotations:
- for xpos, ypos, text in zip(xticks.keys(), _df['upper'].values, annotations):
- ax.text(xpos, ypos*1.15, text, horizontalalignment='center', rotation='vertical', fontsize='x-small')
- ax.set_xticks(list(xticks.keys()))
-
- if put_labels_in_legend:
- # Update xticks label with substitute labels
- # Insert legend with updated labels that shows correspondence between substitute label and original label
- xtick_values = [letter for letter, label in zip(substitute_labels, xticks.values())]
- xtick_legend = [f'{letter}: {label}' for letter, label in zip(substitute_labels, xticks.values())]
- h, legs = ax.get_legend_handles_labels()
- ax.legend(h, xtick_legend, loc='center left', fontsize='small', bbox_to_anchor=(1, 0.5))
- ax.set_xticklabels(list(xtick_values))
- else:
- if not xticklabels_horizontal_and_wrapped:
- # xticklabels will be vertical and not wrapped
- ax.set_xticklabels(list(xticks.values()), rotation=90)
- else:
- wrapped_labs = ["\n".join(textwrap.wrap(_lab, 20)) for _lab in xticks.values()]
- ax.set_xticklabels(wrapped_labs)
-
- ax.grid(axis="y")
- ax.spines['top'].set_visible(False)
- ax.spines['right'].set_visible(False)
- fig.tight_layout()
-
- return fig, ax
-
- # %% Define parameter names
- param_names = get_parameter_names_from_scenario_file()
-
- # %% Quantify the health gains associated with all interventions combined.
-
- # Absolute Number of Deaths and DALYs
- num_deaths = extract_results(
- results_folder,
- module='tlo.methods.demography',
- key='death',
- custom_generate_series=get_num_deaths,
- do_scaling=True
- ).pipe(set_param_names_as_column_index_level_0)
-
- num_dalys = extract_results(
- results_folder,
- module='tlo.methods.healthburden',
- key='dalys_stacked',
- custom_generate_series=get_num_dalys,
- do_scaling=True
- ).pipe(set_param_names_as_column_index_level_0)
-
- # %% Charts of total numbers of deaths / DALYS
- num_dalys_summarized = summarize(num_dalys).loc[0].unstack().reindex(param_names)
- num_deaths_summarized = summarize(num_deaths).loc[0].unstack().reindex(param_names)
-
- name_of_plot = f'Deaths, {target_period()}'
- fig, ax = do_bar_plot_with_ci(num_deaths_summarized / 1e6)
- ax.set_title(name_of_plot)
- ax.set_ylabel('(Millions)')
- fig.tight_layout()
- ax.axhline(num_deaths_summarized.loc['Baseline', 'mean']/1e6, color='black', alpha=0.5)
- fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', '')))
- fig.show()
- plt.close(fig)
-
- name_of_plot = f'All Scenarios: DALYs, {target_period()}'
- fig, ax = do_bar_plot_with_ci(num_dalys_summarized / 1e6)
- ax.set_title(name_of_plot)
- ax.set_ylabel('(Millions)')
- ax.axhline(num_dalys_summarized.loc['Baseline', 'mean']/1e6, color='black', alpha=0.5)
- fig.tight_layout()
- fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', '')))
- fig.show()
- plt.close(fig)
-
-
- # %% Deaths and DALYS averted relative to Status Quo
- num_deaths_averted = summarize(
- -1.0 *
- pd.DataFrame(
- find_difference_relative_to_comparison_series(
- num_deaths.loc[0],
- comparison='Baseline')
- ).T
- ).iloc[0].unstack().reindex(param_names).drop(['Baseline'])
-
- pc_deaths_averted = 100.0 * summarize(
- -1.0 *
- pd.DataFrame(
- find_difference_relative_to_comparison_series(
- num_deaths.loc[0],
- comparison='Baseline',
- scaled=True)
- ).T
- ).iloc[0].unstack().reindex(param_names).drop(['Baseline'])
-
- num_dalys_averted = summarize(
- -1.0 *
- pd.DataFrame(
- find_difference_relative_to_comparison_series(
- num_dalys.loc[0],
- comparison='Baseline')
- ).T
- ).iloc[0].unstack().reindex(param_names).drop(['Baseline'])
-
- pc_dalys_averted = 100.0 * summarize(
- -1.0 *
- pd.DataFrame(
- find_difference_relative_to_comparison_series(
- num_dalys.loc[0],
- comparison='Baseline',
- scaled=True)
- ).T
- ).iloc[0].unstack().reindex(param_names).drop(['Baseline'])
-
- # DEATHS
- name_of_plot = f'Additional Deaths Averted vs Baseline, {target_period()}'
- fig, ax = do_bar_plot_with_ci(
- num_deaths_averted.clip(lower=0.0),
- annotations=[
- f"{round(row['mean'], 0)} ({round(row['lower'], 1)}-{round(row['upper'], 1)}) %"
- for _, row in pc_deaths_averted.clip(lower=0.0).iterrows()
- ]
- )
- ax.set_title(name_of_plot)
- ax.set_ylabel('Additional Deaths Averted vs Baseline')
- fig.tight_layout()
- fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', '')))
- fig.show()
- plt.close(fig)
-
- # DALYS
- name_of_plot = f'DALYs Averted vs Baseline, {target_period()}'
- fig, ax = do_bar_plot_with_ci(
- (num_dalys_averted / 1e6).clip(lower=0.0),
- annotations=[
- f"{round(row['mean'])} ({round(row['lower'], 1)}-{round(row['upper'], 1)}) %"
- for _, row in pc_dalys_averted.clip(lower=0.0).iterrows()
- ]
- )
- ax.set_title(name_of_plot)
- ax.set_ylabel('Additional DALYS Averted vs Baseline \n(Millions)')
- fig.tight_layout()
- fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', '')))
- fig.show()
- plt.close(fig)
-
-
- # %% DALYS averted relative to Baseline - broken down by major cause (HIV, TB, MALARIA)
-
- def get_total_num_dalys_by_label(_df):
- """Return the total number of DALYS in the TARGET_PERIOD by wealth and cause label."""
- y = _df \
- .loc[_df['year'].between(*[d.year for d in TARGET_PERIOD])] \
- .drop(columns=['date', 'year', 'li_wealth']) \
- .sum(axis=0)
-
- # define course cause mapper for HIV, TB, MALARIA and OTHER
- causes = {
- 'AIDS': 'HIV/AIDS',
- 'TB (non-AIDS)': 'TB',
- 'Malaria': 'Malaria',
- '': 'Other', # defined in order to use this dict to determine ordering of the causes in output
- }
- causes_relabels = y.index.map(causes).fillna('Other')
-
- return y.groupby(by=causes_relabels).sum()[list(causes.values())]
-
- total_num_dalys_by_label_results = extract_results(
- results_folder,
- module="tlo.methods.healthburden",
- key="dalys_by_wealth_stacked_by_age_and_time",
- custom_generate_series=get_total_num_dalys_by_label,
- do_scaling=True,
- ).pipe(set_param_names_as_column_index_level_0)
-
- total_num_dalys_by_label_results_averted_vs_baseline = summarize(
- -1.0 * find_difference_relative_to_comparison_series_dataframe(
- total_num_dalys_by_label_results,
- comparison='Baseline'
- ),
- only_mean=True
- )
-
- # Check that when we sum across the causes, we get the same total as calculated when we didn't split by cause.
- assert (
- (total_num_dalys_by_label_results_averted_vs_baseline.sum(axis=0).sort_index()
- - num_dalys_averted['mean'].sort_index()
- ) < 1e-6
- ).all()
-
- # Make a separate plot for the scale-up of each program/programs
- plots = {
- 'HIV programs': [
- 'HIV Programs Scale-up WITHOUT HSS PACKAGE',
- 'HIV Programs Scale-up WITH HSS PACKAGE',
- ],
- 'TB programs': [
- 'TB Programs Scale-up WITHOUT HSS PACKAGE',
- 'TB Programs Scale-up WITH HSS PACKAGE',
- ],
- 'Malaria programs': [
- 'Malaria Programs Scale-up WITHOUT HSS PACKAGE',
- 'Malaria Programs Scale-up WITH HSS PACKAGE',
- ],
- 'All programs': [
- 'FULL HSS PACKAGE',
- 'HIV/Tb/Malaria Programs Scale-up WITHOUT HSS PACKAGE',
- 'HIV/Tb/Malaria Programs Scale-up WITH HSS PACKAGE',
- ]
- }
-
- for plot_name, scenario_names in plots.items():
- name_of_plot = f'{plot_name}'
- fig, ax = plt.subplots()
- total_num_dalys_by_label_results_averted_vs_baseline[scenario_names].T.plot.bar(
- stacked=True,
- ax=ax,
- rot=0,
- alpha=0.75
- )
- ax.set_ylim([0, 10e7])
- ax.set_title(name_of_plot)
- ax.set_ylabel(f'DALYs Averted vs Baseline, {target_period()}\n(Millions)')
- wrapped_labs = ["\n".join(textwrap.wrap(_lab.get_text(), 20)) for _lab in ax.get_xticklabels()]
- ax.set_xticklabels(wrapped_labs)
- fig.tight_layout()
- fig.savefig(make_graph_file_name(name_of_plot.replace(' ', '_').replace(',', '')))
- fig.show()
- plt.close(fig)
-
- # todo: Neaten graphs
- # todo: other metrics of health
- # todo: other graphs, broken down by age/sex (this can also be cribbed from overview paper stuff)
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser()
- parser.add_argument("results_folder", type=Path) # outputs/horizontal_and_vertical_programs-2024-05-16
- args = parser.parse_args()
-
- apply(
- results_folder=args.results_folder,
- output_folder=args.results_folder,
- resourcefilepath=Path('./resources')
- )
diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/gf_analyses/cost_analysis_hss_elements_gf.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/gf_analyses/cost_analysis_hss_elements_gf.py
new file mode 100644
index 0000000000..7d3274343d
--- /dev/null
+++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/gf_analyses/cost_analysis_hss_elements_gf.py
@@ -0,0 +1,468 @@
+"""Produce plots to show the impact each the healthcare system (overall health impact) when running under different
+scenarios (scenario_impact_of_healthsystem.py)
+
+with reduced consumables logging
+/Users/tmangal/PycharmProjects/TLOmodel/outputs/t.mangal@imperial.ac.uk/hss_elements-2024-11-12T172311Z
+"""
+
+from pathlib import Path
+from tlo import Date
+
+import datetime
+import os
+import textwrap
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+import numpy as np
+import pandas as pd
+
+from tlo.analysis.utils import (
+ extract_params,
+ extract_results,
+ get_scenario_info,
+ get_scenario_outputs,
+ load_pickled_dataframes,
+ summarize
+)
+
+from scripts.costing.cost_estimation import (estimate_input_cost_of_scenarios,
+ summarize_cost_data,
+ do_stacked_bar_plot_of_cost_by_category,
+ do_line_plot_of_cost,
+ generate_multiple_scenarios_roi_plot,
+ estimate_projected_health_spending)
+
+# Define a timestamp for script outputs
+timestamp = datetime.datetime.now().strftime("_%Y_%m_%d_%H_%M")
+
+# Print the start time of the script
+print('Script Start', datetime.datetime.now().strftime('%H:%M'))
+
+# Create folders to store results
+resourcefilepath = Path("./resources")
+outputfilepath = Path('./outputs/t.mangal@imperial.ac.uk')
+figurespath = Path('./outputs/global_fund_roi_analysis/hss_elements/')
+if not os.path.exists(figurespath):
+ os.makedirs(figurespath)
+roi_outputs_folder_gf = Path(figurespath / 'gf/roi')
+if not os.path.exists(roi_outputs_folder_gf):
+ os.makedirs(roi_outputs_folder_gf)
+roi_outputs_folder_fcdo = Path(figurespath / 'fcdo/roi')
+if not os.path.exists(roi_outputs_folder_fcdo):
+ os.makedirs(roi_outputs_folder_fcdo)
+
+# Load result files
+# ------------------------------------------------------------------------------------------------------------------
+results_folder = get_scenario_outputs('hss_elements-2024-11-12T172311Z.py', outputfilepath)[0]
+
+# Check can read results from draw=0, run=0
+log = load_pickled_dataframes(results_folder, 0, 0) # look at one log (so can decide what to extract)
+params = extract_params(results_folder)
+
+# Declare default parameters for cost analysis
+# ------------------------------------------------------------------------------------------------------------------
+# Period relevant for costing
+TARGET_PERIOD_INTERVENTION = (Date(2025, 1, 1), Date(2035, 12, 31)) # This is the period that is costed
+relevant_period_for_costing = [i.year for i in TARGET_PERIOD_INTERVENTION]
+list_of_relevant_years_for_costing = list(range(relevant_period_for_costing[0], relevant_period_for_costing[1] + 1))
+
+# Scenarios
+hss_scenarios = {0: "Baseline", 1: "HRH Moderate Scale-up (1%)", 2: "HRH Scale-up Following Historical Growth", 3: "HRH Accelerated Scale-up (6%)",
+ 4: "Increase Capacity at Primary Care Levels", 5: "Consumables Increased to 75th Percentile",
+ 6: "Consumables Available at HIV levels", 7: "Consumables Available at EPI levels", 8: "HSS PACKAGE: Realistic expansion"}
+hs_scenarios_substitutedict = {0:"0", 1: "A", 2: "B", 3: "C",
+4: "D", 5: "E", 6: "F",
+7: "G", 8: "H"}
+hss_scenarios_for_report = [0, 1, 2, 3, 4, 5, 6, 7, 8]
+color_map = {
+ 'Baseline': '#a50026',
+ 'HRH Moderate Scale-up (1%)': '#d73027',
+ 'HRH Scale-up Following Historical Growth': '#f46d43',
+ 'HRH Accelerated Scale-up (6%)': '#fdae61',
+ 'Increase Capacity at Primary Care Levels': '#fee08b',
+ 'Consumables Increased to 75th Percentile': '#d9ef8b',
+ 'Consumables Available at HIV levels': '#a6d96a',
+ 'Consumables Available at EPI levels': '#66bd63',
+ 'HSS PACKAGE: Realistic expansion': '#3288bd'
+}
+
+# Cost-effectiveness threshold
+chosen_cet = 199.620811947318 # This is based on the estimate from Lomas et al (2023)- $160.595987085533 in 2019 USD coverted to 2023 USD
+# based on Ochalek et al (2018) - the paper provided the value $61 in 2016 USD terms, this value is $77.4 in 2023 USD terms
+chosen_value_of_statistical_life = 834
+
+# Discount rate
+discount_rate = 0.03
+
+# Define a function to create bar plots
+def do_bar_plot_with_ci(_df, annotations=None, xticklabels_horizontal_and_wrapped=False):
+ """Make a vertical bar plot for each row of _df, using the columns to identify the height of the bar and the
+ extent of the error bar."""
+
+ # Calculate y-error bars
+ yerr = np.array([
+ (_df['mean'] - _df['lower']).values,
+ (_df['upper'] - _df['mean']).values,
+ ])
+
+ # Map xticks based on the hss_scenarios dictionary
+ xticks = {index: hss_scenarios.get(index, f"Scenario {index}") for index in _df.index}
+
+ # Retrieve colors from color_map based on the xticks labels
+ colors = [color_map.get(label, '#333333') for label in xticks.values()] # default to grey if not found
+
+ # Generate consecutive x positions for the bars, ensuring no gaps
+ x_positions = np.arange(len(xticks)) # Consecutive integers for each bar position
+
+ fig, ax = plt.subplots()
+ ax.bar(
+ x_positions,
+ _df['mean'].values,
+ yerr=yerr,
+ color=colors, # Set bar colors
+ alpha=1,
+ ecolor='black',
+ capsize=10,
+ )
+
+ # Add optional annotations above each bar
+ if annotations:
+ for xpos, ypos, text in zip(x_positions, _df['upper'].values, annotations):
+ ax.text(xpos, ypos * 1.05, text, horizontalalignment='center', fontsize=8)
+
+ # Set x-tick labels with wrapped text if required
+ wrapped_labs = ["\n".join(textwrap.wrap(label, 25)) for label in xticks.values()]
+ ax.set_xticks(x_positions) # Set x-ticks to consecutive positions
+ ax.set_xticklabels(wrapped_labs, rotation=45 if not xticklabels_horizontal_and_wrapped else 0, ha='right',
+ fontsize=8)
+
+ # Set y-axis limit to upper max + 500
+ ax.set_ylim(_df['lower'].min()*1.25, _df['upper'].max()*1.25)
+
+ # Set font size for y-tick labels and grid
+ ax.tick_params(axis='y', labelsize=9)
+ ax.tick_params(axis='x', labelsize=9)
+
+ ax.grid(axis="y")
+ ax.spines['top'].set_visible(False)
+ ax.spines['right'].set_visible(False)
+ fig.tight_layout()
+
+ return fig, ax
+
+def do_standard_bar_plot_with_ci(_df, set_colors=None, annotations=None,
+ xticklabels_horizontal_and_wrapped=False,
+ put_labels_in_legend=True,
+ offset=1e6):
+ """Make a vertical bar plot for each row of _df, using the columns to identify the height of the bar and the
+ extent of the error bar."""
+
+ substitute_labels = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+
+ yerr = np.array([
+ (_df['mean'] - _df['lower']).values,
+ (_df['upper'] - _df['mean']).values,
+ ])
+# TODO should be above be 'median'
+ xticks = {(i + 0.5): k for i, k in enumerate(_df.index)}
+
+ if set_colors:
+ colors = [color_map.get(series, 'grey') for series in _df.index]
+ else:
+ cmap = sns.color_palette('Spectral', as_cmap=True)
+ rescale = lambda y: (y - np.min(y)) / (np.max(y) - np.min(y)) # noqa: E731
+ colors = list(map(cmap, rescale(np.array(list(xticks.keys()))))) if put_labels_in_legend else None
+
+ fig, ax = plt.subplots(figsize=(10, 5))
+ ax.bar(
+ xticks.keys(),
+ _df['mean'].values,
+ yerr=yerr,
+ ecolor='black',
+ color=colors,
+ capsize=10,
+ label=xticks.values()
+ )
+
+ if annotations:
+ for xpos, (ypos, text) in zip(xticks.keys(), zip(_df['upper'].values.flatten(), annotations)):
+ annotation_y = ypos + offset
+
+ ax.text(
+ xpos,
+ annotation_y,
+ '\n'.join(text.split(' ', 1)),
+ horizontalalignment='center',
+ verticalalignment='bottom', # Aligns text at the bottom of the annotation position
+ fontsize='x-small',
+ rotation='horizontal'
+ )
+
+ ax.set_xticks(list(xticks.keys()))
+
+ if put_labels_in_legend:
+ # Update xticks label with substitute labels
+ # Insert legend with updated labels that shows correspondence between substitute label and original label
+ # Use htm_scenarios for the legend
+ xtick_legend = [f'{letter}: {hss_scenarios.get(label, label)}' for letter, label in zip(substitute_labels, xticks.values())]
+ xtick_values = [letter for letter, label in zip(substitute_labels, xticks.values())]
+
+ h, legs = ax.get_legend_handles_labels()
+ ax.legend(h, xtick_legend, loc='center left', fontsize='small', bbox_to_anchor=(1, 0.5))
+ ax.set_xticklabels(xtick_values)
+ else:
+ if not xticklabels_horizontal_and_wrapped:
+ # xticklabels will be vertical and not wrapped
+ ax.set_xticklabels(list(xticks.values()), rotation=90)
+ else:
+ wrapped_labs = ["\n".join(textwrap.wrap(_lab, 20)) for _lab in xticks.values()]
+ ax.set_xticklabels(wrapped_labs)
+
+ # Extend ylim to accommodate data labels
+ ymin, ymax = ax.get_ylim()
+ extension = 0.1 * (ymax - ymin) # 10% of range
+ ax.set_ylim(ymin - extension, ymax + extension) # Set new y-axis limits with the extended range
+
+ ax.grid(axis="y")
+ ax.spines['top'].set_visible(False)
+ ax.spines['right'].set_visible(False)
+ #fig.tight_layout()
+ fig.tight_layout(pad=2.0)
+ plt.subplots_adjust(left=0.15, right=0.85) # Adjust left and right margins
+
+ return fig, ax
+
+# Estimate standard input costs of scenario
+# -----------------------------------------------------------------------------------------------------------------------
+input_costs = estimate_input_cost_of_scenarios(results_folder, resourcefilepath,
+ _years=list_of_relevant_years_for_costing, cost_only_used_staff=True,
+ _discount_rate = discount_rate)
+# _draws = htm_scenarios_for_gf_report --> this subset is created after calculating malaria scale up costs
+# TODO Remove the manual fix below once the logging for these is corrected
+input_costs.loc[input_costs.cost_subgroup == 'Oxygen, 1000 liters, primarily with oxygen cylinders', 'cost'] = \
+ input_costs.loc[input_costs.cost_subgroup == 'Oxygen, 1000 liters, primarily with oxygen cylinders', 'cost']/10
+#input_costs.loc[input_costs.cost_subgroup == 'Depot-Medroxyprogesterone Acetate 150 mg - 3 monthly', 'cost'] =\
+# input_costs.loc[input_costs.cost_subgroup == 'Depot-Medroxyprogesterone Acetate 150 mg - 3 monthly', 'cost']/7
+#input_costs = apply_discounting_to_cost_data(input_costs, _discount_rate = discount_rate)
+
+# %%
+# Return on Invesment analysis
+# Calculate incremental cost
+# -----------------------------------------------------------------------------------------------------------------------
+# Aggregate input costs for further analysis
+input_costs_subset = input_costs[
+ (input_costs['year'] >= relevant_period_for_costing[0]) & (input_costs['year'] <= relevant_period_for_costing[1])]
+# TODO the above step may not longer be needed
+total_input_cost = input_costs_subset.groupby(['draw', 'run'])['cost'].sum()
+total_input_cost_summarized = summarize_cost_data(total_input_cost.unstack(level='run'))
+def find_difference_relative_to_comparison(_ser: pd.Series,
+ comparison: str,
+ scaled: bool = False,
+ drop_comparison: bool = True,
+ ):
+ """Find the difference in the values in a pd.Series with a multi-index, between the draws (level 0)
+ within the runs (level 1), relative to where draw = `comparison`.
+ The comparison is `X - COMPARISON`."""
+ return _ser \
+ .unstack(level=0) \
+ .apply(lambda x: (x - x[comparison]) / (x[comparison] if scaled else 1.0), axis=1) \
+ .drop(columns=([comparison] if drop_comparison else [])) \
+ .stack()
+
+
+incremental_scenario_cost = (pd.DataFrame(
+ find_difference_relative_to_comparison(
+ total_input_cost,
+ comparison=0) # sets the comparator to 0 which is the Actual scenario
+).T.iloc[0].unstack()).T
+
+# Monetary value of health impact
+# -----------------------------------------------------------------------------------------------------------------------
+def get_num_dalys(_df):
+ """Return total number of DALYS (Stacked) by label (total within the TARGET_PERIOD).
+ Throw error if not a record for every year in the TARGET PERIOD (to guard against inadvertently using
+ results from runs that crashed mid-way through the simulation.
+ """
+ years_needed = relevant_period_for_costing # [i.year for i in TARGET_PERIOD_INTERVENTION]
+ assert set(_df.year.unique()).issuperset(years_needed), "Some years are not recorded."
+ _df = _df.loc[_df.year.between(*years_needed)].drop(columns=['date', 'sex', 'age_range']).groupby('year').sum().sum(axis = 1)
+
+ # Initial year and discount rate
+ initial_year = min(_df.index.unique())
+
+ # Calculate the discounted values
+ discounted_values = _df / (1 + discount_rate) ** (_df.index - initial_year)
+
+ return pd.Series(discounted_values.sum())
+
+num_dalys = extract_results(
+ results_folder,
+ module='tlo.methods.healthburden',
+ key='dalys_stacked',
+ custom_generate_series=get_num_dalys,
+ do_scaling=True
+)
+
+# Get absolute DALYs averted
+num_dalys_averted = (-1.0 *
+ pd.DataFrame(
+ find_difference_relative_to_comparison(
+ num_dalys.loc[0],
+ comparison=0) # sets the comparator to 0 which is the Actual scenario
+ ).T.iloc[0].unstack(level='run'))
+num_dalys_averted_fcdo_scenarios = num_dalys_averted[
+ num_dalys_averted.index.get_level_values(0).isin(hss_scenarios_for_report)]
+num_dalys_averted_gf_scenarios = num_dalys_averted[
+ num_dalys_averted.index.get_level_values(0).isin(hss_scenarios_for_report)]
+
+# The monetary value of the health benefit is delta health times CET (negative values are set to 0)
+def get_monetary_value_of_incremental_health(_num_dalys_averted, _chosen_value_of_life_year):
+ monetary_value_of_incremental_health = (_num_dalys_averted * _chosen_value_of_life_year).clip(lower=0.0)
+ return monetary_value_of_incremental_health
+
+# TODO check that the above calculation is correct
+
+# 3. Return on Investment Plot
+# ----------------------------------------------------
+projected_health_spending = estimate_projected_health_spending(resourcefilepath,
+ results_folder,
+ _years = list_of_relevant_years_for_costing,
+ _discount_rate = discount_rate,
+ _summarize = True)
+projected_health_spending_baseline = projected_health_spending[projected_health_spending.index.get_level_values(0) == 0]['mean'][0]
+
+# FCDO
+# Combined ROI plot of relevant scenarios
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [1,2,3,4],
+ _scenario_dict = hss_scenarios,
+ _outputfilepath=roi_outputs_folder_fcdo,
+ _value_of_life_suffix = 'HR_VSL',
+ _year_suffix= f' ({str(relevant_period_for_costing[0])} - {str(relevant_period_for_costing[1])})',
+ _projected_health_spending = projected_health_spending_baseline)
+
+# Combined ROI plot of relevant scenarios
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [5,6,7],
+ _scenario_dict = hss_scenarios,
+ _outputfilepath=roi_outputs_folder_fcdo,
+ _value_of_life_suffix = 'Consumables_VSL',
+ _y_axis_lim= 50,
+ _year_suffix= f' ({str(relevant_period_for_costing[0])} - {str(relevant_period_for_costing[1])})',
+ _projected_health_spending = projected_health_spending_baseline)
+
+# Combined ROI plot of relevant scenarios
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [8],
+ _scenario_dict = hss_scenarios,
+ _outputfilepath=roi_outputs_folder_fcdo,
+ _value_of_life_suffix = 'HSS_VSL',
+ _year_suffix= f' ({str(relevant_period_for_costing[0])} - {str(relevant_period_for_costing[1])})',
+ _projected_health_spending = projected_health_spending_baseline)
+
+# 4. Plot Maximum ability-to-pay at CET
+# ----------------------------------------------------
+max_ability_to_pay_for_implementation = (get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_cet) - incremental_scenario_cost).clip(
+ lower=0.0) # monetary value - change in costs
+max_ability_to_pay_for_implementation_summarized = summarize_cost_data(max_ability_to_pay_for_implementation)
+max_ability_to_pay_for_implementation_summarized_fcdo = max_ability_to_pay_for_implementation_summarized[
+ max_ability_to_pay_for_implementation_summarized.index.get_level_values(0).isin(hss_scenarios_for_report)]
+max_ability_to_pay_for_implementation_summarized_gf = max_ability_to_pay_for_implementation_summarized[
+ max_ability_to_pay_for_implementation_summarized.index.get_level_values(0).isin(hss_scenarios_for_report)]
+
+# FCDO
+# Plot Maximum ability to pay
+name_of_plot = f'Maximum ability to pay at CET, {relevant_period_for_costing[0]}-{relevant_period_for_costing[1]}'
+fig, ax = do_standard_bar_plot_with_ci(
+ (max_ability_to_pay_for_implementation_summarized_fcdo / 1e6),
+ annotations=[
+ f"{row['mean'] / projected_health_spending_baseline :.2%} ({row['lower'] / projected_health_spending_baseline :.2%}- {row['upper'] / projected_health_spending_baseline:.2%})"
+ for _, row in max_ability_to_pay_for_implementation_summarized_fcdo.iterrows()
+ ],
+ xticklabels_horizontal_and_wrapped=False,
+ put_labels_in_legend=True,
+ offset=50,
+)
+ax.set_title(name_of_plot)
+ax.set_ylabel('Maximum ability to pay \n(Millions)')
+ax.set_ylim(bottom=0)
+fig.tight_layout()
+fig.savefig(roi_outputs_folder_fcdo / name_of_plot.replace(' ', '_').replace(',', ''))
+plt.close(fig)
+
+# Plot incremental costs
+incremental_scenario_cost_summarized = summarize_cost_data(incremental_scenario_cost)
+# Keep only scenarios of interest
+incremental_scenario_cost_summarized_fcdo = incremental_scenario_cost_summarized[
+ incremental_scenario_cost_summarized.index.get_level_values(0).isin(hss_scenarios_for_report)]
+incremental_scenario_cost_summarized_gf = incremental_scenario_cost_summarized[
+ incremental_scenario_cost_summarized.index.get_level_values(0).isin(hss_scenarios_for_report)]
+
+# FCDO
+name_of_plot = f'Incremental scenario cost relative to baseline {relevant_period_for_costing[0]}-{relevant_period_for_costing[1]}'
+fig, ax = do_standard_bar_plot_with_ci(
+ (incremental_scenario_cost_summarized_fcdo / 1e6),
+ annotations=[
+ f"{row['mean'] / projected_health_spending_baseline :.2%} ({row['lower'] / projected_health_spending_baseline :.2%}- {row['upper'] / projected_health_spending_baseline:.2%})"
+ for _, row in incremental_scenario_cost_summarized_fcdo.iterrows()
+ ],
+ xticklabels_horizontal_and_wrapped=False,
+ put_labels_in_legend=True,
+ offset=50,
+)
+ax.set_title(name_of_plot)
+ax.set_ylabel('Cost \n(USD Millions)')
+ax.set_ylim(bottom=0)
+fig.tight_layout()
+fig.savefig(roi_outputs_folder_fcdo / name_of_plot.replace(' ', '_').replace(',', ''))
+plt.close(fig)
+
+# 4. Plot costs
+# ----------------------------------------------------
+# FCDO
+input_costs_for_plot = input_costs_subset[input_costs_subset.draw.isin(hss_scenarios_for_report)]
+# First summarize all input costs
+input_costs_for_plot_summarized = input_costs_for_plot.groupby(['draw', 'year', 'cost_subcategory', 'Facility_Level', 'cost_subgroup', 'cost_category']).agg(
+ mean=('cost', 'mean'),
+ lower=('cost', lambda x: x.quantile(0.025)),
+ upper=('cost', lambda x: x.quantile(0.975))
+).reset_index()
+input_costs_for_plot_summarized = input_costs_for_plot_summarized.melt(
+ id_vars=['draw', 'year', 'cost_subcategory', 'Facility_Level', 'cost_subgroup', 'cost_category'],
+ value_vars=['mean', 'lower', 'upper'],
+ var_name='stat',
+ value_name='cost'
+)
+
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'all', _disaggregate_by_subgroup = False, _outputfilepath = Path(figurespath / 'fcdo'), _scenario_dict = hs_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'all', _year = [2025], _disaggregate_by_subgroup = False, _outputfilepath = Path(figurespath / 'fcdo'), _scenario_dict = hs_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'human resources for health', _disaggregate_by_subgroup = False, _outputfilepath = Path(figurespath / 'fcdo'), _scenario_dict = hs_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'medical consumables', _disaggregate_by_subgroup = False, _outputfilepath = Path(figurespath / 'fcdo'), _scenario_dict = hs_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'medical equipment', _disaggregate_by_subgroup = False, _outputfilepath = Path(figurespath / 'fcdo'), _scenario_dict = hs_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'other', _disaggregate_by_subgroup = False, _outputfilepath = Path(figurespath / 'fcdo'), _scenario_dict = hs_scenarios_substitutedict)
+
+# Global Fund
+input_costs_for_plot = input_costs_subset[input_costs_subset.draw.isin(hss_scenarios_for_report)]
+# First summarize all input costs
+input_costs_for_plot_summarized = input_costs_for_plot.groupby(['draw', 'year', 'cost_subcategory', 'Facility_Level', 'cost_subgroup', 'cost_category']).agg(
+ mean=('cost', 'mean'),
+ lower=('cost', lambda x: x.quantile(0.025)),
+ upper=('cost', lambda x: x.quantile(0.975))
+).reset_index()
+input_costs_for_plot_summarized = input_costs_for_plot_summarized.melt(
+ id_vars=['draw', 'year', 'cost_subcategory', 'Facility_Level', 'cost_subgroup', 'cost_category'],
+ value_vars=['mean', 'lower', 'upper'],
+ var_name='stat',
+ value_name='cost'
+)
+
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'all', _disaggregate_by_subgroup = False, _outputfilepath = Path(figurespath / 'gf'), _scenario_dict = hs_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'all', _year = [2025], _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'human resources for health', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'medical consumables', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'medical equipment', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'other', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict)
diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/gf_analyses/cost_analysis_htm_with_and_without_hss_gf.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/gf_analyses/cost_analysis_htm_with_and_without_hss_gf.py
new file mode 100644
index 0000000000..1840d47cae
--- /dev/null
+++ b/src/scripts/comparison_of_horizontal_and_vertical_programs/gf_analyses/cost_analysis_htm_with_and_without_hss_gf.py
@@ -0,0 +1,692 @@
+"""Produce plots to show the impact each the healthcare system (overall health impact) when running under different
+scenarios (scenario_vertical_programs_with_and_without_hss.py)
+
+job ID:
+results for FCDO and GF presentations Sept 2024:
+htm_with_and_without_hss-2024-09-04T143044Z
+
+results for updates 30Sept2024 (IRS in high-risk distr and reduced gen pop RDT):
+htm_with_and_without_hss-2024-09-17T083150Z
+
+with reduced consumables logging
+htm_with_and_without_hss-2024-11-12T172503Z
+"""
+
+from pathlib import Path
+from tlo import Date
+
+import datetime
+import os
+import textwrap
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+import numpy as np
+import pandas as pd
+
+from tlo.analysis.utils import (
+ extract_params,
+ extract_results,
+ get_scenario_info,
+ get_scenario_outputs,
+ load_pickled_dataframes,
+ summarize
+)
+
+from scripts.costing.cost_estimation import (estimate_input_cost_of_scenarios,
+ summarize_cost_data,
+ do_stacked_bar_plot_of_cost_by_category,
+ do_line_plot_of_cost,
+ generate_roi_plots,
+ generate_multiple_scenarios_roi_plot,
+ estimate_projected_health_spending,
+ apply_discounting_to_cost_data)
+
+# Define a timestamp for script outputs
+timestamp = datetime.datetime.now().strftime("_%Y_%m_%d_%H_%M")
+
+# Print the start time of the script
+print('Script Start', datetime.datetime.now().strftime('%H:%M'))
+
+# Create folders to store results
+resourcefilepath = Path("./resources")
+outputfilepath = Path('./outputs/t.mangal@imperial.ac.uk')
+figurespath = Path('./outputs/global_fund_roi_analysis/htm_with_and_without_hss')
+if not os.path.exists(figurespath):
+ os.makedirs(figurespath)
+roi_outputs_folder = Path(figurespath / 'roi')
+if not os.path.exists(roi_outputs_folder):
+ os.makedirs(roi_outputs_folder)
+
+# Load result files
+#------------------------------------------------------------------------------------------------------------------
+results_folder = get_scenario_outputs('htm_with_and_without_hss-2024-11-12T172503Z.py', outputfilepath)[0]
+
+# Check can read results from draw=0, run=0
+log = load_pickled_dataframes(results_folder, 0, 0) # look at one log (so can decide what to extract)
+params = extract_params(results_folder)
+info = get_scenario_info(results_folder)
+
+# Declare default parameters for cost analysis
+#------------------------------------------------------------------------------------------------------------------
+# Population scaling factor for malaria scale-up projections
+population_scaling_factor = log['tlo.methods.demography']['scaling_factor']['scaling_factor'].iloc[0]
+# Load the list of districts and their IDs
+district_dict = pd.read_csv(resourcefilepath / 'demography' / 'ResourceFile_Population_2010.csv')[
+ ['District_Num', 'District']].drop_duplicates()
+district_dict = dict(zip(district_dict['District_Num'], district_dict['District']))
+
+# Period relevant for costing
+TARGET_PERIOD= (Date(2025, 1, 1), Date(2035, 12, 31)) # This is the period that is costed
+relevant_period_for_costing = [i.year for i in TARGET_PERIOD]
+list_of_relevant_years_for_costing = list(range(relevant_period_for_costing[0], relevant_period_for_costing[1] + 1))
+
+# Scenarios
+htm_scenarios = {0:"Baseline", 1: "HSS PACKAGE: Realistic", 2: "HIV Programs Scale-up WITHOUT HSS PACKAGE",
+3: "HIV Programs Scale-up WITH REALISTIC HSS PACKAGE", 4: "TB Programs Scale-up WITHOUT HSS PACKAGE",
+5: "TB Programs Scale-up WITH REALISTIC HSS PACKAGE", 6: "Malaria Programs Scale-up WITHOUT HSS PACKAGE",
+7: "Malaria Programs Scale-up WITH REALISTIC HSS PACKAGE", 8: "HTM Programs Scale-up WITHOUT HSS PACKAGE",
+9: "HTM Programs Scale-up WITH REALISTIC HSS PACKAGE", 10: "HTM Programs Scale-up WITH SUPPLY CHAINS", 11: "HTM Programs Scale-up WITH HRH"}
+
+htm_scenarios_substitutedict = {0:"0", 1: "A", 2: "B", 3: "C",
+4: "D", 5: "E", 6: "F",
+7: "G", 8: "H", 9: "I",
+10: "J", 11: "K"}
+
+# Subset of scenarios included in analysis
+htm_scenarios_for_report = list(range(0,12))
+
+color_map = {
+ 'Baseline': '#9e0142',
+ 'HSS PACKAGE: Realistic': '#d8434e',
+ 'HIV Programs Scale-up WITHOUT HSS PACKAGE': '#f36b48',
+ 'HIV Programs Scale-up WITH REALISTIC HSS PACKAGE': '#fca45c',
+ 'TB Programs Scale-up WITHOUT HSS PACKAGE': '#fddc89',
+ 'TB Programs Scale-up WITH REALISTIC HSS PACKAGE': '#e7f7a0',
+ 'Malaria Programs Scale-up WITHOUT HSS PACKAGE': '#a5dc97',
+ 'Malaria Programs Scale-up WITH REALISTIC HSS PACKAGE': '#6dc0a6',
+ 'HTM Programs Scale-up WITHOUT HSS PACKAGE': '#438fba',
+ 'HTM Programs Scale-up WITH REALISTIC HSS PACKAGE': '#5e4fa2',
+ 'HTM Programs Scale-up WITH SUPPLY CHAINS': '#3c71aa',
+ 'HTM Programs Scale-up WITH HRH': '#2f6094',
+}
+
+# Cost-effectiveness threshold
+chosen_cet = 199.620811947318 # This is based on the estimate from Lomas et al (2023)- $160.595987085533 in 2019 USD coverted to 2023 USD
+# based on Ochalek et al (2018) - the paper provided the value $61 in 2016 USD terms, this value is $77.4 in 2023 USD terms
+chosen_value_of_statistical_life = 834
+
+# Discount rate
+discount_rate = 0.03
+
+# Define a function to create bar plots
+def do_bar_plot_with_ci(_df, annotations=None, xticklabels_horizontal_and_wrapped=False):
+ """Make a vertical bar plot for each row of _df, using the columns to identify the height of the bar and the
+ extent of the error bar."""
+
+ # Calculate y-error bars
+ yerr = np.array([
+ (_df['mean'] - _df['lower']).values,
+ (_df['upper'] - _df['mean']).values,
+ ])
+
+ # Map xticks based on the hss_scenarios dictionary
+ xticks = {index: htm_scenarios.get(index, f"Scenario {index}") for index in _df.index}
+
+ # Retrieve colors from color_map based on the xticks labels
+ colors = [color_map.get(label, '#333333') for label in xticks.values()] # default to grey if not found
+
+ # Generate consecutive x positions for the bars, ensuring no gaps
+ x_positions = np.arange(len(xticks)) # Consecutive integers for each bar position
+
+ fig, ax = plt.subplots()
+ ax.bar(
+ x_positions,
+ _df['mean'].values,
+ yerr=yerr,
+ color=colors, # Set bar colors
+ alpha=1,
+ ecolor='black',
+ capsize=10,
+ )
+
+ # Add optional annotations above each bar
+ if annotations:
+ for xpos, ypos, text in zip(x_positions, _df['upper'].values, annotations):
+ ax.text(xpos, ypos * 1.05, text, horizontalalignment='center', fontsize=8)
+
+ # Set x-tick labels with wrapped text if required
+ wrapped_labs = ["\n".join(textwrap.wrap(label,30)) for label in xticks.values()]
+ ax.set_xticks(x_positions) # Set x-ticks to consecutive positions
+ ax.set_xticklabels(wrapped_labs, rotation=45 if not xticklabels_horizontal_and_wrapped else 0, ha='right',
+ fontsize=7)
+
+ # Set y-axis limit to upper max + 500
+ ax.set_ylim(_df['lower'].min()*1.25, _df['upper'].max()*1.25)
+
+ # Set font size for y-tick labels and grid
+ ax.tick_params(axis='y', labelsize=9)
+ ax.tick_params(axis='x', labelsize=9)
+
+ ax.grid(axis="y")
+ ax.spines['top'].set_visible(False)
+ ax.spines['right'].set_visible(False)
+ fig.tight_layout()
+
+ return fig, ax
+
+def do_standard_bar_plot_with_ci(_df, set_colors=None, annotations=None,
+ xticklabels_horizontal_and_wrapped=False,
+ put_labels_in_legend=True,
+ offset=1e6):
+ """Make a vertical bar plot for each row of _df, using the columns to identify the height of the bar and the
+ extent of the error bar."""
+
+ substitute_labels = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+
+ yerr = np.array([
+ (_df['mean'] - _df['lower']).values,
+ (_df['upper'] - _df['mean']).values,
+ ])
+# TODO should be above be 'median'
+ xticks = {(i + 0.5): k for i, k in enumerate(_df.index)}
+
+ if set_colors:
+ colors = [color_map.get(series, 'grey') for series in _df.index]
+ else:
+ cmap = sns.color_palette('Spectral', as_cmap=True)
+ rescale = lambda y: (y - np.min(y)) / (np.max(y) - np.min(y)) # noqa: E731
+ colors = list(map(cmap, rescale(np.array(list(xticks.keys()))))) if put_labels_in_legend else None
+
+ fig, ax = plt.subplots(figsize=(10, 5))
+ ax.bar(
+ xticks.keys(),
+ _df['mean'].values,
+ yerr=yerr,
+ ecolor='black',
+ color=colors,
+ capsize=10,
+ label=xticks.values()
+ )
+
+ if annotations:
+ for xpos, (ypos, text) in zip(xticks.keys(), zip(_df['upper'].values.flatten(), annotations)):
+ annotation_y = ypos + offset
+
+ ax.text(
+ xpos,
+ annotation_y,
+ '\n'.join(text.split(' ', 1)),
+ horizontalalignment='center',
+ verticalalignment='bottom', # Aligns text at the bottom of the annotation position
+ fontsize='x-small',
+ rotation='horizontal'
+ )
+
+ ax.set_xticks(list(xticks.keys()))
+
+ if put_labels_in_legend:
+ # Update xticks label with substitute labels
+ # Insert legend with updated labels that shows correspondence between substitute label and original label
+ # Use htm_scenarios for the legend
+ xtick_legend = [f'{letter}: {htm_scenarios.get(label, label)}' for letter, label in zip(substitute_labels, xticks.values())]
+ xtick_values = [letter for letter, label in zip(substitute_labels, xticks.values())]
+
+ h, legs = ax.get_legend_handles_labels()
+ ax.legend(h, xtick_legend, loc='center left', fontsize='small', bbox_to_anchor=(1, 0.5))
+ ax.set_xticklabels(xtick_values)
+ else:
+ if not xticklabels_horizontal_and_wrapped:
+ # xticklabels will be vertical and not wrapped
+ ax.set_xticklabels(list(xticks.values()), rotation=90)
+ else:
+ wrapped_labs = ["\n".join(textwrap.wrap(_lab, 20)) for _lab in xticks.values()]
+ ax.set_xticklabels(wrapped_labs)
+
+ # Extend ylim to accommodate data labels
+ ymin, ymax = ax.get_ylim()
+ extension = 0.1 * (ymax - ymin) # 10% of range
+ ax.set_ylim(ymin - extension, ymax + extension) # Set new y-axis limits with the extended range
+
+ ax.grid(axis="y")
+ ax.spines['top'].set_visible(False)
+ ax.spines['right'].set_visible(False)
+ #fig.tight_layout()
+ fig.tight_layout(pad=2.0)
+ plt.subplots_adjust(left=0.15, right=0.85) # Adjust left and right margins
+
+ return fig, ax
+
+# Estimate standard input costs of scenario
+#-----------------------------------------------------------------------------------------------------------------------
+input_costs = estimate_input_cost_of_scenarios(results_folder, resourcefilepath,
+ _years= list_of_relevant_years_for_costing, cost_only_used_staff= True,
+ _discount_rate = discount_rate)
+
+# TODO Remove the manual fix below once the logging for these is corrected
+# Post-run fixes to costs due to challenges with calibration
+input_costs.loc[input_costs.cost_subgroup == 'Oxygen, 1000 liters, primarily with oxygen cylinders', 'cost'] = \
+ input_costs.loc[input_costs.cost_subgroup == 'Oxygen, 1000 liters, primarily with oxygen cylinders', 'cost']/10
+#input_costs = apply_discounting_to_cost_data(input_costs, _discount_rate = discount_rate)
+
+# Add additional costs pertaining to simulation (Only for scenarios with Malaria scale-up)
+#-----------------------------------------------------------------------------------------------------------------------
+# Extract supply chain cost as a proportion of consumable costs to apply to malaria scale-up commodities
+# Load primary costing resourcefile
+workbook_cost = pd.read_excel((resourcefilepath / "costing/ResourceFile_Costing.xlsx"),
+ sheet_name=None)
+# Read parameters for consumables costs
+# Load consumables cost data
+unit_price_consumable = workbook_cost["consumables"]
+unit_price_consumable = unit_price_consumable.rename(columns=unit_price_consumable.iloc[0])
+unit_price_consumable = unit_price_consumable[['Item_Code', 'Final_price_per_chosen_unit (USD, 2023)']].reset_index(
+ drop=True).iloc[1:]
+unit_price_consumable = unit_price_consumable[unit_price_consumable['Item_Code'].notna()]
+
+# Assume that the cost of procurement, warehousing and distribution is a fixed proportion of consumable purchase costs
+# The fixed proportion is based on Resource Mapping Expenditure data from 2018
+resource_mapping_data = workbook_cost["resource_mapping_r7_summary"]
+# Make sure values are numeric
+expenditure_column = ['EXPENDITURE (USD) (Jul 2018 - Jun 2019)']
+resource_mapping_data[expenditure_column] = resource_mapping_data[expenditure_column].apply(
+ lambda x: pd.to_numeric(x, errors='coerce'))
+supply_chain_expenditure = \
+resource_mapping_data[resource_mapping_data['Cost Type'] == 'Supply Chain'][expenditure_column].sum()[0]
+consumables_purchase_expenditure = \
+resource_mapping_data[resource_mapping_data['Cost Type'] == 'Drugs and Commodities'][expenditure_column].sum()[0] + \
+resource_mapping_data[resource_mapping_data['Cost Type'] == 'HIV Drugs and Commodities'][expenditure_column].sum()[0]
+supply_chain_cost_proportion = supply_chain_expenditure / consumables_purchase_expenditure
+
+# In this case malaria intervention scale-up costs were not included in the standard estimate_input_cost_of_scenarios function
+list_of_draws_with_malaria_scaleup_parameters = params[(params.module_param == 'Malaria:scaleup_start_year')]
+list_of_draws_with_malaria_scaleup_parameters.loc[:,'value'] = pd.to_numeric(list_of_draws_with_malaria_scaleup_parameters['value'])
+list_of_draws_with_malaria_scaleup_implemented_in_costing_period = list_of_draws_with_malaria_scaleup_parameters[(list_of_draws_with_malaria_scaleup_parameters['value'] < max(relevant_period_for_costing))].index.to_list()
+
+# 1. IRS costs
+irs_coverage_rate = 0.8
+districts_with_irs_scaleup = ['Kasungu', 'Mchinji', 'Lilongwe', 'Lilongwe City', 'Dowa', 'Ntchisi', 'Salima', 'Mangochi',
+ 'Mwanza', 'Likoma', 'Nkhotakota']
+# Convert above list of district names to numeric district identifiers
+district_keys_with_irs_scaleup = [key for key, name in district_dict.items() if name in districts_with_irs_scaleup]
+year_of_malaria_scaleup_start = list_of_draws_with_malaria_scaleup_parameters.loc[:,'value'].reset_index()['value'][0]
+final_year_for_costing = max(list_of_relevant_years_for_costing)
+TARGET_PERIOD_MALARIA_SCALEUP = (Date(year_of_malaria_scaleup_start, 1, 1), Date(final_year_for_costing, 12, 31))
+
+# Get population by district
+def get_total_population_by_district(_df):
+ years_needed = [i.year for i in TARGET_PERIOD_MALARIA_SCALEUP] # we only consider the population for the malaria scale-up period
+ # because those are the years relevant for malaria scale-up costing
+ _df['year'] = pd.to_datetime(_df['date']).dt.year
+ assert set(_df.year.unique()).issuperset(years_needed), "Some years are not recorded."
+ _df = pd.melt(_df.drop(columns = 'date'), id_vars = ['year']).rename(columns = {'variable': 'district'})
+ return pd.Series(
+ data=_df
+ .loc[_df.year.between(*years_needed)]
+ .set_index(['year', 'district'])['value']
+ )
+
+district_population_by_year = extract_results(
+ results_folder,
+ module='tlo.methods.malaria',
+ key='pop_district',
+ custom_generate_series=get_total_population_by_district,
+ do_scaling=True
+)
+
+def get_number_of_people_covered_by_malaria_scaleup(_df, list_of_districts_covered = None, draws_included = None):
+ _df = pd.DataFrame(_df)
+ # Reset the index to make 'district' a column
+ _df = _df.reset_index()
+ # Convert the 'district' column to numeric values
+ _df['district'] = pd.to_numeric(_df['district'], errors='coerce')
+ _df = _df.set_index(['year', 'district'])
+ if list_of_districts_covered is not None:
+ _df.loc[~_df.index.get_level_values('district').isin(list_of_districts_covered), :] = 0
+ if draws_included is not None:
+ _df.loc[:, ~_df.columns.get_level_values('draw').isin(draws_included)] = 0
+ return _df
+
+district_population_covered_by_irs_scaleup_by_year = get_number_of_people_covered_by_malaria_scaleup(district_population_by_year,
+ list_of_districts_covered=district_keys_with_irs_scaleup,
+ draws_included = list_of_draws_with_malaria_scaleup_implemented_in_costing_period)
+
+irs_cost_per_person = unit_price_consumable[unit_price_consumable.Item_Code == 161]['Final_price_per_chosen_unit (USD, 2023)']
+# This cost includes non-consumable costs - personnel, equipment, fuel, logistics and planning, shipping, PPE. The cost is measured per person protected. Based on Stelmach et al (2018)
+irs_multiplication_factor = irs_cost_per_person * irs_coverage_rate
+total_irs_cost = irs_multiplication_factor.iloc[0] * district_population_covered_by_irs_scaleup_by_year # for districts and scenarios included
+total_irs_cost = total_irs_cost.groupby(level='year').sum()
+
+# 2. Bednet costs
+bednet_coverage_rate = 0.7
+# We can assume 3-year lifespan of a bednet, each bednet covering 1.8 people.
+inflation_2011_to_2023 = 1.35
+unit_cost_of_bednet = unit_price_consumable[unit_price_consumable.Item_Code == 160]['Final_price_per_chosen_unit (USD, 2023)'] + (8.27 - 3.36) * inflation_2011_to_2023
+# Stelmach et al Tanzania https://pmc.ncbi.nlm.nih.gov/articles/PMC6169190/#_ad93_ (Price in 2011 USD) - This cost includes non-consumable costs - personnel, equipment, fuel, logistics and planning, shipping. The cost is measured per net distributed
+# Note that the cost per net of $3.36 has been replaced with a cost of Malawi Kwacha 667 (2023) as per the Central Medical Stores Trust sales catalogue
+
+# We add supply chain costs (procurement + distribution + warehousing) because the unit_cost does not include this
+annual_bednet_cost_per_person = unit_cost_of_bednet / 1.8 / 3
+bednet_multiplication_factor = bednet_coverage_rate * annual_bednet_cost_per_person
+
+district_population_covered_by_bednet_scaleup_by_year = get_number_of_people_covered_by_malaria_scaleup(district_population_by_year,
+ draws_included = list_of_draws_with_malaria_scaleup_implemented_in_costing_period) # All districts covered
+
+total_bednet_cost = bednet_multiplication_factor.iloc[0] * district_population_covered_by_bednet_scaleup_by_year # for scenarios included
+total_bednet_cost = total_bednet_cost.groupby(level='year').sum()
+
+# Malaria scale-up costs - TOTAL
+malaria_scaleup_costs = [
+ (total_irs_cost.reset_index(), 'cost_of_IRS_scaleup'),
+ (total_bednet_cost.reset_index(), 'cost_of_bednet_scaleup'),
+]
+def melt_and_label_malaria_scaleup_cost(_df, label):
+ multi_index = pd.MultiIndex.from_tuples(_df.columns)
+ _df.columns = multi_index
+
+ # reshape dataframe and assign 'draw' and 'run' as the correct column headers
+ melted_df = pd.melt(_df, id_vars=['year']).rename(columns={'variable_0': 'draw', 'variable_1': 'run'})
+ # Replace item_code with consumable_name_tlo
+ melted_df['cost_subcategory'] = label
+ melted_df['cost_category'] = 'other'
+ melted_df['cost_subgroup'] = 'NA'
+ melted_df['Facility_Level'] = 'all'
+ melted_df = melted_df.rename(columns={'value': 'cost'})
+ return melted_df
+
+# Iterate through additional costs, melt and concatenate
+for df, label in malaria_scaleup_costs:
+ new_df = melt_and_label_malaria_scaleup_cost(df, label)
+ new_df = apply_discounting_to_cost_data(new_df, _discount_rate= discount_rate, _year = relevant_period_for_costing[0])
+ input_costs = pd.concat([input_costs, new_df], ignore_index=True)
+
+# Extract input_costs for browsing
+input_costs.groupby(['draw', 'run', 'cost_category', 'cost_subcategory', 'cost_subgroup','year'])['cost'].sum().to_csv(figurespath / 'cost_detailed.csv')
+
+# %%
+# Return on Invesment analysis
+# Calculate incremental cost
+# -----------------------------------------------------------------------------------------------------------------------
+# Aggregate input costs for further analysis (this step is needed because the malaria specific scale-up costs start from the year or malaria scale-up implementation)
+input_costs_subset = input_costs[
+ (input_costs['year'] >= relevant_period_for_costing[0]) & (input_costs['year'] <= relevant_period_for_costing[1])]
+
+# Extract input_costs for TGF
+input_costs_subset.groupby(['draw', 'run', 'cost_category', 'year'])['cost'].sum().to_csv(figurespath / 'cost_for_gf.csv')
+
+total_input_cost = input_costs_subset.groupby(['draw', 'run'])['cost'].sum()
+total_input_cost_summarized = summarize_cost_data(total_input_cost.unstack(level='run'))
+def find_difference_relative_to_comparison(_ser: pd.Series,
+ comparison: str,
+ scaled: bool = False,
+ drop_comparison: bool = True,
+ ):
+ """Find the difference in the values in a pd.Series with a multi-index, between the draws (level 0)
+ within the runs (level 1), relative to where draw = `comparison`.
+ The comparison is `X - COMPARISON`."""
+ return _ser \
+ .unstack(level=0) \
+ .apply(lambda x: (x - x[comparison]) / (x[comparison] if scaled else 1.0), axis=1) \
+ .drop(columns=([comparison] if drop_comparison else [])) \
+ .stack()
+
+
+incremental_scenario_cost = (pd.DataFrame(
+ find_difference_relative_to_comparison(
+ total_input_cost,
+ comparison=0) # sets the comparator to 0 which is the Actual scenario
+).T.iloc[0].unstack()).T
+
+# Keep only scenarios of interest
+incremental_scenario_cost = incremental_scenario_cost[
+ incremental_scenario_cost.index.get_level_values(0).isin(htm_scenarios_for_report)]
+
+# Monetary value of health impact
+# -----------------------------------------------------------------------------------------------------------------------
+def get_num_dalys(_df):
+ """Return total number of DALYS (Stacked) by label (total within the TARGET_PERIOD).
+ Throw error if not a record for every year in the TARGET PERIOD (to guard against inadvertently using
+ results from runs that crashed mid-way through the simulation.
+ """
+ years_needed = relevant_period_for_costing
+ assert set(_df.year.unique()).issuperset(years_needed), "Some years are not recorded."
+ _df = _df.loc[_df.year.between(*years_needed)].drop(columns=['date', 'sex', 'age_range']).groupby('year').sum().sum(axis = 1)
+
+ # Initial year and discount rate
+ initial_year = min(_df.index.unique())
+
+ # Calculate the discounted values
+ discounted_values = _df / (1 + discount_rate) ** (_df.index - initial_year)
+
+ return pd.Series(discounted_values.sum())
+
+num_dalys = extract_results(
+ results_folder,
+ module='tlo.methods.healthburden',
+ key='dalys_stacked',
+ custom_generate_series=get_num_dalys,
+ do_scaling=True
+)
+
+# Get absolute DALYs averted
+num_dalys_averted = (-1.0 *
+ pd.DataFrame(
+ find_difference_relative_to_comparison(
+ num_dalys.loc[0],
+ comparison=0) # sets the comparator to 0 which is the Actual scenario
+ ).T.iloc[0].unstack(level='run'))
+num_dalys_averted = num_dalys_averted[num_dalys_averted.index.get_level_values(0).isin(htm_scenarios_for_report)]
+
+# The monetary value of the health benefit is delta health times CET (negative values are set to 0)
+def get_monetary_value_of_incremental_health(_num_dalys_averted, _chosen_value_of_life_year):
+ monetary_value_of_incremental_health = (_num_dalys_averted * _chosen_value_of_life_year).clip(lower=0.0)
+ return monetary_value_of_incremental_health
+
+# TODO check that the above calculation is correct
+
+# 3. Return on Investment Plot
+# ----------------------------------------------------
+projected_health_spending = estimate_projected_health_spending(resourcefilepath,
+ results_folder,
+ _years = list_of_relevant_years_for_costing,
+ _discount_rate = discount_rate,
+ _summarize = True)
+projected_health_spending_baseline = projected_health_spending[projected_health_spending.index.get_level_values(0) == 0]['mean'][0]
+
+# Combined ROI plot of relevant scenarios
+'''
+# HTM scenarios X 5
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [1,8,9,10,11],
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _value_of_life_suffix = 'all_HTM_VSL')
+
+# HTM scenarios X 3
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [1,8,9],
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _value_of_life_suffix = 'HTM_full_HSS_VSL')
+
+'''
+
+# Only HSS
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [8,1],
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _value_of_life_suffix = 'HTM_full_HSS_VSL',
+ _plot_vertical_lines_at = [0, 1e9, 3e9],
+ _year_suffix= f' ({str(relevant_period_for_costing[0])} - {str(relevant_period_for_costing[1])})',
+ _projected_health_spending = projected_health_spending_baseline)
+
+# HTM scenarios with HSS
+draw_colors = {8: '#438FBA', 9:'#5E4FA2'}
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [8,9],
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _value_of_life_suffix = 'HTM_full_HSS_VSL',
+ _plot_vertical_lines_at = [0, 1e9, 3e9],
+ _year_suffix= f' ({str(relevant_period_for_costing[0])}- {str(relevant_period_for_costing[1])})',
+ _projected_health_spending = projected_health_spending_baseline,
+ _draw_colors = draw_colors)
+
+# HIV scenarios
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [2,3],
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _year_suffix=f' ({str(relevant_period_for_costing[0])}- {str(relevant_period_for_costing[1])})',
+ _value_of_life_suffix = 'HIV_VSL')
+
+# TB scenarios
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [4,5],
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _year_suffix=f' ({str(relevant_period_for_costing[0])}- {str(relevant_period_for_costing[1])})',
+ _value_of_life_suffix = 'TB_VSL',
+ _y_axis_lim = 30)
+
+# Malaria scenarios
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [6,7],
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _year_suffix=f' ({str(relevant_period_for_costing[0])}- {str(relevant_period_for_costing[1])})',
+ _value_of_life_suffix = 'Malaria_VSL')
+
+# 4. Plot Maximum ability-to-pay at CET
+# ----------------------------------------------------
+max_ability_to_pay_for_implementation = (get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_cet) - incremental_scenario_cost).clip(
+ lower=0.0) # monetary value - change in costs
+max_ability_to_pay_for_implementation_summarized = summarize_cost_data(max_ability_to_pay_for_implementation)
+max_ability_to_pay_for_implementation_summarized = max_ability_to_pay_for_implementation_summarized[
+ max_ability_to_pay_for_implementation_summarized.index.get_level_values(0).isin(htm_scenarios_for_report)]
+
+# Plot Maximum ability to pay
+name_of_plot = f'Maximum ability to pay at CET, {relevant_period_for_costing[0]}-{relevant_period_for_costing[1]}'
+fig, ax = do_standard_bar_plot_with_ci(
+ (max_ability_to_pay_for_implementation_summarized / 1e6),
+ annotations=[
+ f"{row['mean'] / projected_health_spending_baseline :.2%} ({row['lower'] / projected_health_spending_baseline :.2%}- \n {row['upper'] / projected_health_spending_baseline:.2%})"
+ for _, row in max_ability_to_pay_for_implementation_summarized.iterrows()
+ ],
+ xticklabels_horizontal_and_wrapped=False,
+ put_labels_in_legend=True,
+ offset=50,
+)
+ax.set_title(name_of_plot)
+ax.set_ylabel('Maximum ability to pay \n(Millions)')
+ax.set_ylim(bottom=0)
+fig.tight_layout()
+fig.savefig(roi_outputs_folder / name_of_plot.replace(' ', '_').replace(',', ''))
+plt.close(fig)
+
+# Plot incremental costs
+incremental_scenario_cost_summarized = summarize_cost_data(incremental_scenario_cost)
+name_of_plot = f'Incremental scenario cost relative to baseline {relevant_period_for_costing[0]}-{relevant_period_for_costing[1]}'
+fig, ax = do_standard_bar_plot_with_ci(
+ (incremental_scenario_cost_summarized / 1e6),
+ annotations=[
+ f"{row['mean'] / projected_health_spending_baseline :.2%} ({row['lower'] / projected_health_spending_baseline :.2%}- {row['upper'] / projected_health_spending_baseline:.2%})"
+ for _, row in incremental_scenario_cost_summarized.iterrows()
+ ],
+ xticklabels_horizontal_and_wrapped=False,
+ put_labels_in_legend=True,
+ offset=50,
+)
+ax.set_title(name_of_plot)
+ax.set_ylabel('Cost \n(USD Millions)')
+ax.set_ylim(bottom=0)
+fig.tight_layout()
+fig.savefig(roi_outputs_folder / name_of_plot.replace(' ', '_').replace(',', ''))
+plt.close(fig)
+
+# 4. Plot costs
+# ----------------------------------------------------
+input_costs_for_plot = input_costs_subset[input_costs_subset.draw.isin(htm_scenarios_for_report)]
+# First summarize all input costs
+input_costs_for_plot_summarized = input_costs_for_plot.groupby(['draw', 'year', 'cost_subcategory', 'Facility_Level', 'cost_subgroup', 'cost_category']).agg(
+ mean=('cost', 'mean'),
+ lower=('cost', lambda x: x.quantile(0.025)),
+ upper=('cost', lambda x: x.quantile(0.975))
+).reset_index()
+input_costs_for_plot_summarized = input_costs_for_plot_summarized.melt(
+ id_vars=['draw', 'year', 'cost_subcategory', 'Facility_Level', 'cost_subgroup', 'cost_category'],
+ value_vars=['mean', 'lower', 'upper'],
+ var_name='stat',
+ value_name='cost'
+)
+
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'all', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = htm_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'all', _year = [2025], _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = htm_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'human resources for health', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = htm_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'medical consumables', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = htm_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'medical equipment', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = htm_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'other', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = htm_scenarios_substitutedict)
+
+'''
+# Population size across scenarios
+def get_total_population_by_age_range(_df):
+ years_needed = [i.year for i in TARGET_PERIOD] # we only consider the population for the malaria scale-up period
+ _df['year'] = pd.to_datetime(_df['date']).dt.year
+ assert set(_df.year.unique()).issuperset(years_needed), "Some years are not recorded."
+ _df = pd.melt(_df.drop(columns = 'date'), id_vars = ['year']).rename(columns = {'variable': 'age_range'})
+ return pd.Series(
+ data=_df
+ .loc[_df.year.between(*years_needed)]
+ .groupby('age_range')['value'].sum()
+ )
+
+male_population_by_age_range = summarize(extract_results(
+ results_folder,
+ module='tlo.methods.demography',
+ key='age_range_m',
+ custom_generate_series=get_total_population_by_age_range,
+ do_scaling=True
+))
+
+female_population_by_age_range = summarize(extract_results(
+ results_folder,
+ module='tlo.methods.demography',
+ key='age_range_f',
+ custom_generate_series=get_total_population_by_age_range,
+ do_scaling=True
+))
+
+# Plot male population (age 0-4)
+name_of_plot = f'Male population, 0-4 years, {relevant_period_for_costing[0]}-{relevant_period_for_costing[1]}'
+fig, ax = do_standard_bar_plot_with_ci(
+ (male_population_by_age_range[male_population_by_age_range.index.get_level_values('age_range') == '0-4'].sum()/1e6).unstack(),
+ xticklabels_horizontal_and_wrapped=False,
+)
+ax.set_title(name_of_plot)
+ax.set_ylabel('Population \n(Millions)')
+fig.tight_layout()
+fig.savefig(roi_outputs_folder / name_of_plot.replace(' ', '_').replace(',', ''))
+plt.close(fig)
+
+
+# Plot female population (age 0-4)
+name_of_plot = f'Female population, 0-4 years, {relevant_period_for_costing[0]}-{relevant_period_for_costing[1]}'
+fig, ax = do_standard_bar_plot_with_ci(
+ (female_population_by_age_range[female_population_by_age_range.index.get_level_values('age_range') == '0-4'].sum()/1e6).unstack(),
+ xticklabels_horizontal_and_wrapped=False,
+)
+ax.set_title(name_of_plot)
+ax.set_ylabel('Population \n(Millions)')
+fig.tight_layout()
+fig.savefig(roi_outputs_folder / name_of_plot.replace(' ', '_').replace(',', ''))
+plt.close(fig)
+
+'''
+
diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/mini_analysis_for_testing/mini_version_scenario.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/mini_analysis_for_testing/mini_version_scenario.py
deleted file mode 100644
index 24256efd3a..0000000000
--- a/src/scripts/comparison_of_horizontal_and_vertical_programs/mini_analysis_for_testing/mini_version_scenario.py
+++ /dev/null
@@ -1,85 +0,0 @@
-"""This Scenario file is intended to help with debugging the scale-up of HIV. Tb and Malaria services, per issue #1413.
-
-Changes to the main analysis:
-
-* We're running this in MODE 1 and we're only looking.
-* We're capturing the logged output from HIV, Tb and malaria
-* We're limiting it to few scenarios: baseline + the scale-up of all HTM programs (no HealthSystem scale-up)
-
-"""
-
-from pathlib import Path
-from typing import Dict
-
-from scripts.comparison_of_horizontal_and_vertical_programs.scenario_definitions import (
- ScenarioDefinitions,
-)
-from tlo import Date, logging
-from tlo.analysis.utils import mix_scenarios
-from tlo.methods.fullmodel import fullmodel
-from tlo.methods.scenario_switcher import ImprovedHealthSystemAndCareSeekingScenarioSwitcher
-from tlo.scenario import BaseScenario
-
-
-class MiniRunHTMWithAndWithoutHSS(BaseScenario):
- def __init__(self):
- super().__init__()
- self.seed = 0
- self.start_date = Date(2010, 1, 1)
- self.end_date = Date(2031, 1, 1)
- self.pop_size = 100_000
- self._scenarios = self._get_scenarios()
- self.number_of_draws = len(self._scenarios)
- self.runs_per_draw = 1
-
- def log_configuration(self):
- return {
- 'filename': 'mini_htm_with_and_without_hss',
- 'directory': Path('./outputs'),
- 'custom_levels': {
- '*': logging.WARNING,
- 'tlo.methods.demography': logging.INFO,
- 'tlo.methods.demography.detail': logging.WARNING,
- 'tlo.methods.healthburden': logging.INFO,
- 'tlo.methods.healthsystem': logging.WARNING,
- 'tlo.methods.healthsystem.summary': logging.INFO,
- 'tlo.methods.hiv': logging.INFO,
- 'tlo.methods.tb': logging.INFO,
- 'tlo.methods.malaria': logging.INFO,
- }
- }
-
- def modules(self):
- return (
- fullmodel(resourcefilepath=self.resources)
- + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)]
- )
-
- def draw_parameters(self, draw_number, rng):
- if draw_number < len(self._scenarios):
- return list(self._scenarios.values())[draw_number]
-
- def _get_scenarios(self) -> Dict[str, Dict]:
- """Return the Dict with values for the parameters that are changed, keyed by a name for the scenario."""
- # Load helper class containing the definitions of the elements of all the scenarios
- scenario_definitions = ScenarioDefinitions()
-
- return {
- "Baseline":
- scenario_definitions.baseline(),
-
- # - - - HIV & TB & MALARIA SCALE-UP WITHOUT HSS PACKAGE- - -
- "HIV/Tb/Malaria Programs Scale-up WITHOUT HSS PACKAGE":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.hiv_scaleup(),
- scenario_definitions.tb_scaleup(),
- scenario_definitions.malaria_scaleup(),
- ),
- }
-
-
-if __name__ == '__main__':
- from tlo.cli import scenario_run
-
- scenario_run([__file__])
diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py
deleted file mode 100644
index 31615bdc27..0000000000
--- a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_definitions.py
+++ /dev/null
@@ -1,150 +0,0 @@
-"""The file contains all the definitions of scenarios used the Horizontal and Vertical Program Impact Analyses"""
-from typing import Dict
-
-from tlo.analysis.utils import get_parameters_for_status_quo, mix_scenarios
-
-
-class ScenarioDefinitions:
-
- @property
- def YEAR_OF_CHANGE_FOR_HSS(self) -> int:
- """Year in which Health Systems Strengthening changes are made."""
- return 2019 # <-- baseline year of Human Resources for Health is 2018, and this is consistent with calibration
- # during 2015-2019 period.
-
- @property
- def YEAR_OF_CHANGE_FOR_HTM(self) -> int:
- """Year in which HIV, TB, Malaria scale-up changes are made."""
- return 2019
-
- def baseline(self) -> Dict:
- """Return the Dict with values for the parameter changes that define the baseline scenario. """
- return mix_scenarios(
- get_parameters_for_status_quo(), # <-- Parameters that have been the calibration targets
-
- # Set up the HealthSystem to transition from Mode 1 -> Mode 2, with rescaling when there are HSS changes
- {
- "HealthSystem": {
- "mode_appt_constraints": 1, # <-- Mode 1 prior to change to preserve calibration
- "mode_appt_constraints_postSwitch": 2, # <-- Mode 2 post-change to show effects of HRH
- "scale_to_effective_capabilities": True,
- # <-- Transition into Mode2 with the effective capabilities in HRH 'revealed' in Mode 1
- "year_mode_switch": self.YEAR_OF_CHANGE_FOR_HSS,
-
- # Normalize the behaviour of Mode 2
- "policy_name": "Naive",
- "tclose_overwrite": 1,
- "tclose_days_offset_overwrite": 7,
- }
- },
- )
-
- def double_capacity_at_primary_care(self) -> Dict:
- return {
- 'HealthSystem': {
- 'year_HR_scaling_by_level_and_officer_type': self.YEAR_OF_CHANGE_FOR_HSS,
- 'HR_scaling_by_level_and_officer_type_mode': 'x2_fac0&1',
- }
- }
-
- def hrh_at_pop_grwoth(self) -> Dict:
- return {
- 'HealthSystem': {
- 'yearly_HR_scaling_mode': 'scaling_by_population_growth',
- # This is in-line with population growth _after 2018_ (baseline year for HRH)
- }
- }
-
- def hrh_at_gdp_growth(self) -> Dict:
- return {
- 'HealthSystem': {
- 'yearly_HR_scaling_mode': 'GDP_growth',
- # This is GDP growth after 2018 (baseline year for HRH)
- }
- }
-
- def hrh_above_gdp_growth(self) -> Dict:
- return {
- 'HealthSystem': {
- 'yearly_HR_scaling_mode': 'GDP_growth_fHE_case5',
- # This is above-GDP growth after 2018 (baseline year for HRH)
- }
- }
-
- def perfect_clinical_practices(self) -> Dict:
- return {
- 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': {
- 'max_healthsystem_function': [False, True], # <-- switch from False to True mid-way
- 'year_of_switch': self.YEAR_OF_CHANGE_FOR_HSS,
- }
- }
-
- def perfect_healthcare_seeking(self) -> Dict:
- return {
- 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': {
- 'max_healthcare_seeking': [False, True], # <-- switch from False to True mid-way
- 'year_of_switch': self.YEAR_OF_CHANGE_FOR_HSS,
- }
- }
-
- def vital_items_available(self) -> Dict:
- return {
- 'HealthSystem': {
- 'year_cons_availability_switch': self.YEAR_OF_CHANGE_FOR_HSS,
- 'cons_availability_postSwitch': 'all_vital_available',
- }
- }
-
- def medicines_available(self) -> Dict:
- return {
- 'HealthSystem': {
- 'year_cons_availability_switch': self.YEAR_OF_CHANGE_FOR_HSS,
- 'cons_availability_postSwitch': 'all_medicines_available',
- }
- }
-
- def all_consumables_available(self) -> Dict:
- return {
- 'HealthSystem': {
- 'year_cons_availability_switch': self.YEAR_OF_CHANGE_FOR_HSS,
- 'cons_availability_postSwitch': 'all',
- }
- }
-
- def hss_package(self) -> Dict:
- """The parameters for the Health System Strengthening Package"""
- return mix_scenarios(
- self.double_capacity_at_primary_care(), # }
- self.hrh_above_gdp_growth(), # } <-- confirmed that these two do build on one another under
- # mode 2 rescaling: see `test_scaling_up_HRH_using_yearly_scaling_and_scaling_by_level_together`.
- self.perfect_clinical_practices(),
- self.perfect_healthcare_seeking(),
- self.all_consumables_available(),
- )
-
- def hiv_scaleup(self) -> Dict:
- """The parameters for the scale-up of the HIV program"""
- return {
- "Hiv": {
- 'type_of_scaleup': 'max', # <--- using MAXIMUM SCALE-UP as an experiment
- 'scaleup_start_year': self.YEAR_OF_CHANGE_FOR_HTM,
- }
- }
-
- def tb_scaleup(self) -> Dict:
- """The parameters for the scale-up of the TB program"""
- return {
- "Tb": {
- 'type_of_scaleup': 'max', # <--- using MAXIMUM SCALE-UP as an experiment
- 'scaleup_start_year': self.YEAR_OF_CHANGE_FOR_HTM,
- }
- }
-
- def malaria_scaleup(self) -> Dict:
- """The parameters for the scale-up of the Malaria program"""
- return {
- 'Malaria': {
- 'type_of_scaleup': 'max', # <--- using MAXIMUM SCALE-UP as an experiment
- 'scaleup_start_year': self.YEAR_OF_CHANGE_FOR_HTM,
- }
- }
diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_hss_elements.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_hss_elements.py
deleted file mode 100644
index 8c2f2afc09..0000000000
--- a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_hss_elements.py
+++ /dev/null
@@ -1,143 +0,0 @@
-"""This Scenario file run the model under different assumptions for the HealthSystem and Vertical Program Scale-up
-
-Run on the batch system using:
-```
-tlo batch-submit
- src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_hss_elements.py
-```
-
-"""
-
-from pathlib import Path
-from typing import Dict
-
-from scripts.comparison_of_horizontal_and_vertical_programs.scenario_definitions import (
- ScenarioDefinitions,
-)
-from tlo import Date, logging
-from tlo.analysis.utils import mix_scenarios
-from tlo.methods.fullmodel import fullmodel
-from tlo.methods.scenario_switcher import ImprovedHealthSystemAndCareSeekingScenarioSwitcher
-from tlo.scenario import BaseScenario
-
-
-class HSSElements(BaseScenario):
- def __init__(self):
- super().__init__()
- self.seed = 0
- self.start_date = Date(2010, 1, 1)
- self.end_date = Date(2031, 1, 1)
- self.pop_size = 100_000
- self._scenarios = self._get_scenarios()
- self.number_of_draws = len(self._scenarios)
- self.runs_per_draw = 3 # <--- todo: N.B. Very small number of repeated run, to be efficient for now
-
- def log_configuration(self):
- return {
- 'filename': 'hss_elements',
- 'directory': Path('./outputs'),
- 'custom_levels': {
- '*': logging.WARNING,
- 'tlo.methods.demography': logging.INFO,
- 'tlo.methods.demography.detail': logging.WARNING,
- 'tlo.methods.healthburden': logging.INFO,
- 'tlo.methods.healthsystem': logging.WARNING,
- 'tlo.methods.healthsystem.summary': logging.INFO,
- }
- }
-
- def modules(self):
- return (
- fullmodel(resourcefilepath=self.resources)
- + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)]
- )
-
- def draw_parameters(self, draw_number, rng):
- if draw_number < len(self._scenarios):
- return list(self._scenarios.values())[draw_number]
-
- def _get_scenarios(self) -> Dict[str, Dict]:
- """Return the Dict with values for the parameters that are changed, keyed by a name for the scenario."""
-
- scenario_definitions = ScenarioDefinitions()
-
- return {
- "Baseline": scenario_definitions.baseline(),
-
- # ***************************
- # HEALTH SYSTEM STRENGTHENING
- # ***************************
-
- # - - - Human Resource for Health - - -
-
- "Double Capacity at Primary Care":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.double_capacity_at_primary_care(),
- ),
-
- "HRH Keeps Pace with Population Growth":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions._hrh_at_pop_growth(),
- ),
-
- "HRH Increases at GDP Growth":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions._hrh_at_grp_growth(),
- ),
-
- "HRH Increases above GDP Growth":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.hrh_above_gdp_growth(),
- ),
-
-
- # - - - Quality of Care - - -
- "Perfect Clinical Practice":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions._perfect_clinical_practice(),
- ),
-
- "Perfect Healthcare Seeking":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.perfect_healthcare_seeking(),
- ),
-
- # - - - Supply Chains - - -
- "Perfect Availability of Vital Items":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.vital_items_available(),
- ),
-
- "Perfect Availability of Medicines":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.medicines_available(),
-
- ),
-
- "Perfect Availability of All Consumables":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.all_consumables_available(),
- ),
-
- # - - - FULL PACKAGE OF HEALTH SYSTEM STRENGTHENING - - -
- "FULL PACKAGE":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.hss_package(),
- ),
- }
-
-
-if __name__ == '__main__':
- from tlo.cli import scenario_run
-
- scenario_run([__file__])
diff --git a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_vertical_programs_with_and_without_hss.py b/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_vertical_programs_with_and_without_hss.py
deleted file mode 100644
index e4f6dcbd88..0000000000
--- a/src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_vertical_programs_with_and_without_hss.py
+++ /dev/null
@@ -1,147 +0,0 @@
-"""This Scenario file run the model under different assumptions for the HealthSystem and Vertical Program Scale-up
-
-Run on the batch system using:
-```
-tlo batch-submit
- src/scripts/comparison_of_horizontal_and_vertical_programs/scenario_vertical_programs_with_and_without_hss.py
-```
-
-"""
-
-from pathlib import Path
-from typing import Dict
-
-from scripts.comparison_of_horizontal_and_vertical_programs.scenario_definitions import (
- ScenarioDefinitions,
-)
-from tlo import Date, logging
-from tlo.analysis.utils import mix_scenarios
-from tlo.methods.fullmodel import fullmodel
-from tlo.methods.scenario_switcher import ImprovedHealthSystemAndCareSeekingScenarioSwitcher
-from tlo.scenario import BaseScenario
-
-
-class HTMWithAndWithoutHSS(BaseScenario):
- def __init__(self):
- super().__init__()
- self.seed = 0
- self.start_date = Date(2010, 1, 1)
- self.end_date = Date(2031, 1, 1)
- self.pop_size = 100_000
- self._scenarios = self._get_scenarios()
- self.number_of_draws = len(self._scenarios)
- self.runs_per_draw = 3 # <--- todo: N.B. Very small number of repeated run, to be efficient for now
-
- def log_configuration(self):
- return {
- 'filename': 'htm_with_and_without_hss',
- 'directory': Path('./outputs'),
- 'custom_levels': {
- '*': logging.WARNING,
- 'tlo.methods.demography': logging.INFO,
- 'tlo.methods.demography.detail': logging.WARNING,
- 'tlo.methods.healthburden': logging.INFO,
- 'tlo.methods.healthsystem': logging.WARNING,
- 'tlo.methods.healthsystem.summary': logging.INFO,
- 'tlo.methods.hiv': logging.INFO,
- 'tlo.methods.tb': logging.INFO,
- 'tlo.methods.malaria': logging.INFO,
- }
- }
-
- def modules(self):
- return (
- fullmodel(resourcefilepath=self.resources)
- + [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)]
- )
-
- def draw_parameters(self, draw_number, rng):
- if draw_number < len(self._scenarios):
- return list(self._scenarios.values())[draw_number]
-
- def _get_scenarios(self) -> Dict[str, Dict]:
- """Return the Dict with values for the parameters that are changed, keyed by a name for the scenario."""
- # Load helper class containing the definitions of the elements of all the scenarios
- scenario_definitions = ScenarioDefinitions()
-
- return {
- "Baseline":
- scenario_definitions.baseline(),
-
- # - - - FULL PACKAGE OF HEALTH SYSTEM STRENGTHENING - - -
- "FULL HSS PACKAGE":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.hss_package(),
- ),
-
- # **************************************************
- # VERTICAL PROGRAMS WITH AND WITHOUT THE HSS PACKAGE
- # **************************************************
-
- # - - - HIV SCALE-UP WITHOUT HSS PACKAGE- - -
- "HIV Programs Scale-up WITHOUT HSS PACKAGE":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.hiv_scaleup(),
- ),
- # - - - HIV SCALE-UP *WITH* HSS PACKAGE- - -
- "HIV Programs Scale-up WITH HSS PACKAGE":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.hiv_scaleup(),
- scenario_definitions.hss_package(),
- ),
-
- # - - - TB SCALE-UP WITHOUT HSS PACKAGE- - -
- "TB Programs Scale-up WITHOUT HSS PACKAGE":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.tb_scaleup(),
- ),
- # - - - TB SCALE-UP *WITH* HSS PACKAGE- - -
- "TB Programs Scale-up WITH HSS PACKAGE":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.tb_scaleup(),
- scenario_definitions.hss_package(),
- ),
-
- # - - - MALARIA SCALE-UP WITHOUT HSS PACKAGE- - -
- "Malaria Programs Scale-up WITHOUT HSS PACKAGE":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.malaria_scaleup(),
- ),
- # - - - MALARIA SCALE-UP *WITH* HSS PACKAGE- - -
- "Malaria Programs Scale-up WITH HSS PACKAGE":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.malaria_scaleup(),
- scenario_definitions.hss_package(),
- ),
-
- # - - - HIV & TB & MALARIA SCALE-UP WITHOUT HSS PACKAGE- - -
- "HIV/Tb/Malaria Programs Scale-up WITHOUT HSS PACKAGE":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.hiv_scaleup(),
- scenario_definitions.tb_scaleup(),
- scenario_definitions.malaria_scaleup(),
- ),
- # - - - HIV & TB & MALARIA SCALE-UP *WITH* HSS PACKAGE- - -
- "HIV/Tb/Malaria Programs Scale-up WITH HSS PACKAGE":
- mix_scenarios(
- scenario_definitions.baseline(),
- scenario_definitions.hiv_scaleup(),
- scenario_definitions.tb_scaleup(),
- scenario_definitions.malaria_scaleup(),
- scenario_definitions.hss_package(),
- ),
- }
-
-
-if __name__ == '__main__':
- from tlo.cli import scenario_run
-
- scenario_run([__file__])
diff --git a/src/scripts/costing/cost_analysis_hss_elements.py b/src/scripts/costing/cost_analysis_hss_elements.py
new file mode 100644
index 0000000000..1c4decab43
--- /dev/null
+++ b/src/scripts/costing/cost_analysis_hss_elements.py
@@ -0,0 +1,512 @@
+import argparse
+from pathlib import Path
+from tlo import Date
+from collections import Counter, defaultdict
+
+import calendar
+import datetime
+import os
+import textwrap
+
+import matplotlib.pyplot as plt
+from matplotlib.ticker import FuncFormatter
+import numpy as np
+import pandas as pd
+import ast
+import math
+
+from tlo.analysis.utils import (
+ extract_params,
+ extract_results,
+ get_scenario_info,
+ get_scenario_outputs,
+ load_pickled_dataframes,
+ make_age_grp_lookup,
+ make_age_grp_types,
+ summarize,
+ create_pickles_locally,
+ parse_log_file,
+ unflatten_flattened_multi_index_in_logging
+)
+
+from scripts.costing.cost_estimation import (estimate_input_cost_of_scenarios,
+ summarize_cost_data,
+ apply_discounting_to_cost_data,
+ do_stacked_bar_plot_of_cost_by_category,
+ do_line_plot_of_cost,
+ generate_roi_plots,
+ generate_multiple_scenarios_roi_plot)
+
+# Define a timestamp for script outputs
+timestamp = datetime.datetime.now().strftime("_%Y_%m_%d_%H_%M")
+
+# Print the start time of the script
+print('Script Start', datetime.datetime.now().strftime('%H:%M'))
+
+# Create folders to store results
+resourcefilepath = Path("./resources")
+outputfilepath = Path('./outputs/t.mangal@imperial.ac.uk')
+figurespath = Path('./outputs/global_fund_roi_analysis/hss_elements/')
+if not os.path.exists(figurespath):
+ os.makedirs(figurespath)
+roi_outputs_folder_gf = Path(figurespath / 'gf/roi')
+if not os.path.exists(roi_outputs_folder_gf):
+ os.makedirs(roi_outputs_folder_gf)
+roi_outputs_folder_fcdo = Path(figurespath / 'fcdo/roi')
+if not os.path.exists(roi_outputs_folder_fcdo):
+ os.makedirs(roi_outputs_folder_fcdo)
+
+# Load result files
+# ------------------------------------------------------------------------------------------------------------------
+results_folder = get_scenario_outputs('hss_elements-2024-10-22T163857Z.py', outputfilepath)[0]
+
+# Check can read results from draw=0, run=0
+log = load_pickled_dataframes(results_folder, 0, 0) # look at one log (so can decide what to extract)
+params = extract_params(results_folder)
+
+# Declare default parameters for cost analysis
+# ------------------------------------------------------------------------------------------------------------------
+# Period relevant for costing
+TARGET_PERIOD_INTERVENTION = (Date(2025, 1, 1), Date(2035, 12, 31)) # This is the period that is costed
+relevant_period_for_costing = [i.year for i in TARGET_PERIOD_INTERVENTION]
+list_of_relevant_years_for_costing = list(range(relevant_period_for_costing[0], relevant_period_for_costing[1] + 1))
+
+# Scenarios
+hss_scenarios = {0: "Baseline", 1: "HRH Moderate Scale-up (1%)", 2: "HRH Scale-up Following Historical Growth", 3: "HRH Accelerated Scale-up (6%)",
+ 4: "Increase Capacity at Primary Care Levels", 5: "Increase Capacity of CHW", 6: "Consumables Increased to 75th Percentile",
+ 7: "Consumables Available at HIV levels", 8: "Consumables Available at EPI levels", 9: "Perfect Consumables Availability",
+ 10: "HSS PACKAGE: Perfect", 11: "HSS PACKAGE: Realistic expansion, no change in HSB", 12: "HSS PACKAGE: Realistic expansion"}
+hs_scenarios_substitutedict_fcdo = {0:"0", 1: "A", 2: "B", 3: "C",
+4: "D", 5: "5", 6: "E",
+7: "F", 8: "G", 9: "H",
+10: "I", 11: "11", 12: "J"}
+hs_scenarios_substitutedict_gf = {0:"0", 1: "A", 2: "B", 3: "C",
+4: "D", 5: "E", 6: "F",
+7: "G", 8: "H", 9: "9",
+10: "10", 11: "11", 12: "I"}
+hss_scenarios_for_fcdo_report = [0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 12]
+hss_scenarios_for_gf_report = [0, 1, 2, 3, 4, 5, 6, 7, 8, 12]
+color_map = {
+ 'Baseline': '#a50026',
+ 'HRH Moderate Scale-up (1%)': '#d73027',
+ 'HRH Scale-up Following Historical Growth': '#f46d43',
+ 'HRH Accelerated Scale-up (6%)': '#fdae61',
+ 'Increase Capacity at Primary Care Levels': '#fee08b',
+ 'Increase Capacity of CHW': '#ffffbf',
+ 'Consumables Increased to 75th Percentile': '#d9ef8b',
+ 'Consumables Available at HIV levels': '#a6d96a',
+ 'Consumables Available at EPI levels': '#66bd63',
+ 'Perfect Consumables Availability': '#1a9850',
+ 'HSS PACKAGE: Perfect': '#3288bd',
+ 'HSS PACKAGE: Realistic expansion': '#5e4fa2'
+}
+
+# Cost-effectiveness threshold
+chosen_cet = 199.620811947318 # This is based on the estimate from Lomas et al (2023)- $160.595987085533 in 2019 USD coverted to 2023 USD
+# based on Ochalek et al (2018) - the paper provided the value $61 in 2016 USD terms, this value is $77.4 in 2023 USD terms
+chosen_value_of_statistical_life = 834
+
+# Discount rate
+discount_rate = 0.03
+
+# Define a function to create bar plots
+def do_bar_plot_with_ci(_df, annotations=None, xticklabels_horizontal_and_wrapped=False):
+ """Make a vertical bar plot for each row of _df, using the columns to identify the height of the bar and the
+ extent of the error bar."""
+
+ # Calculate y-error bars
+ yerr = np.array([
+ (_df['mean'] - _df['lower']).values,
+ (_df['upper'] - _df['mean']).values,
+ ])
+
+ # Map xticks based on the hss_scenarios dictionary
+ xticks = {index: hss_scenarios.get(index, f"Scenario {index}") for index in _df.index}
+
+ # Retrieve colors from color_map based on the xticks labels
+ colors = [color_map.get(label, '#333333') for label in xticks.values()] # default to grey if not found
+
+ # Generate consecutive x positions for the bars, ensuring no gaps
+ x_positions = np.arange(len(xticks)) # Consecutive integers for each bar position
+
+ fig, ax = plt.subplots()
+ ax.bar(
+ x_positions,
+ _df['mean'].values,
+ yerr=yerr,
+ color=colors, # Set bar colors
+ alpha=1,
+ ecolor='black',
+ capsize=10,
+ )
+
+ # Add optional annotations above each bar
+ if annotations:
+ for xpos, ypos, text in zip(x_positions, _df['upper'].values, annotations):
+ ax.text(xpos, ypos * 1.05, text, horizontalalignment='center', fontsize=8)
+
+ # Set x-tick labels with wrapped text if required
+ wrapped_labs = ["\n".join(textwrap.wrap(label, 25)) for label in xticks.values()]
+ ax.set_xticks(x_positions) # Set x-ticks to consecutive positions
+ ax.set_xticklabels(wrapped_labs, rotation=45 if not xticklabels_horizontal_and_wrapped else 0, ha='right',
+ fontsize=8)
+
+ # Set y-axis limit to upper max + 500
+ ax.set_ylim(_df['lower'].min()*1.25, _df['upper'].max()*1.25)
+
+ # Set font size for y-tick labels and grid
+ ax.tick_params(axis='y', labelsize=9)
+ ax.tick_params(axis='x', labelsize=9)
+
+ ax.grid(axis="y")
+ ax.spines['top'].set_visible(False)
+ ax.spines['right'].set_visible(False)
+ fig.tight_layout()
+
+ return fig, ax
+
+def do_standard_bar_plot_with_ci(_df, set_colors=None, annotations=None,
+ xticklabels_horizontal_and_wrapped=False,
+ put_labels_in_legend=True,
+ offset=1e6):
+ """Make a vertical bar plot for each row of _df, using the columns to identify the height of the bar and the
+ extent of the error bar."""
+
+ substitute_labels = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+
+ yerr = np.array([
+ (_df['mean'] - _df['lower']).values,
+ (_df['upper'] - _df['mean']).values,
+ ])
+# TODO should be above be 'median'
+ xticks = {(i + 0.5): k for i, k in enumerate(_df.index)}
+
+ if set_colors:
+ colors = [color_map.get(series, 'grey') for series in _df.index]
+ else:
+ cmap = sns.color_palette('Spectral', as_cmap=True)
+ rescale = lambda y: (y - np.min(y)) / (np.max(y) - np.min(y)) # noqa: E731
+ colors = list(map(cmap, rescale(np.array(list(xticks.keys()))))) if put_labels_in_legend else None
+
+ fig, ax = plt.subplots(figsize=(10, 5))
+ ax.bar(
+ xticks.keys(),
+ _df['mean'].values,
+ yerr=yerr,
+ ecolor='black',
+ color=colors,
+ capsize=10,
+ label=xticks.values()
+ )
+
+ if annotations:
+ for xpos, (ypos, text) in zip(xticks.keys(), zip(_df['upper'].values.flatten(), annotations)):
+ annotation_y = ypos + offset
+
+ ax.text(
+ xpos,
+ annotation_y,
+ '\n'.join(text.split(' ', 1)),
+ horizontalalignment='center',
+ verticalalignment='bottom', # Aligns text at the bottom of the annotation position
+ fontsize='x-small',
+ rotation='horizontal'
+ )
+
+ ax.set_xticks(list(xticks.keys()))
+
+ if put_labels_in_legend:
+ # Update xticks label with substitute labels
+ # Insert legend with updated labels that shows correspondence between substitute label and original label
+ xtick_values = [letter for letter, label in zip(substitute_labels, xticks.values())]
+ xtick_legend = [f'{letter}: {label}' for letter, label in zip(substitute_labels, xticks.values())]
+ h, legs = ax.get_legend_handles_labels()
+ ax.legend(h, xtick_legend, loc='center left', fontsize='small', bbox_to_anchor=(1, 0.5))
+ ax.set_xticklabels(list(xtick_values))
+ else:
+ if not xticklabels_horizontal_and_wrapped:
+ # xticklabels will be vertical and not wrapped
+ ax.set_xticklabels(list(xticks.values()), rotation=90)
+ else:
+ wrapped_labs = ["\n".join(textwrap.wrap(_lab, 20)) for _lab in xticks.values()]
+ ax.set_xticklabels(wrapped_labs)
+
+ ax.grid(axis="y")
+ ax.spines['top'].set_visible(False)
+ ax.spines['right'].set_visible(False)
+ fig.tight_layout(pad=2.0)
+ plt.subplots_adjust(left=0.15, right=0.85) # Adjust left and right margins
+
+ return fig, ax
+
+# Estimate standard input costs of scenario
+# -----------------------------------------------------------------------------------------------------------------------
+input_costs = estimate_input_cost_of_scenarios(results_folder, resourcefilepath,
+ _years=list_of_relevant_years_for_costing, cost_only_used_staff=True,
+ _discount_rate = discount_rate)
+# _draws = htm_scenarios_for_gf_report --> this subset is created after calculating malaria scale up costs
+# TODO Remove the manual fix below once the logging for these is corrected
+input_costs.loc[input_costs.cost_subgroup == 'Oxygen, 1000 liters, primarily with oxygen cylinders', 'cost'] = \
+ input_costs.loc[input_costs.cost_subgroup == 'Oxygen, 1000 liters, primarily with oxygen cylinders', 'cost']/10
+input_costs.loc[input_costs.cost_subgroup == 'Depot-Medroxyprogesterone Acetate 150 mg - 3 monthly', 'cost'] =\
+ input_costs.loc[input_costs.cost_subgroup == 'Depot-Medroxyprogesterone Acetate 150 mg - 3 monthly', 'cost']/7
+#input_costs = apply_discounting_to_cost_data(input_costs, _discount_rate = discount_rate)
+
+# %%
+# Return on Invesment analysis
+# Calculate incremental cost
+# -----------------------------------------------------------------------------------------------------------------------
+# Aggregate input costs for further analysis
+input_costs_subset = input_costs[
+ (input_costs['year'] >= relevant_period_for_costing[0]) & (input_costs['year'] <= relevant_period_for_costing[1])]
+# TODO the above step may not longer be needed
+total_input_cost = input_costs_subset.groupby(['draw', 'run'])['cost'].sum()
+total_input_cost_summarized = summarize_cost_data(total_input_cost.unstack(level='run'))
+def find_difference_relative_to_comparison(_ser: pd.Series,
+ comparison: str,
+ scaled: bool = False,
+ drop_comparison: bool = True,
+ ):
+ """Find the difference in the values in a pd.Series with a multi-index, between the draws (level 0)
+ within the runs (level 1), relative to where draw = `comparison`.
+ The comparison is `X - COMPARISON`."""
+ return _ser \
+ .unstack(level=0) \
+ .apply(lambda x: (x - x[comparison]) / (x[comparison] if scaled else 1.0), axis=1) \
+ .drop(columns=([comparison] if drop_comparison else [])) \
+ .stack()
+
+
+incremental_scenario_cost = (pd.DataFrame(
+ find_difference_relative_to_comparison(
+ total_input_cost,
+ comparison=0) # sets the comparator to 0 which is the Actual scenario
+).T.iloc[0].unstack()).T
+
+# Monetary value of health impact
+# -----------------------------------------------------------------------------------------------------------------------
+def get_num_dalys(_df):
+ """Return total number of DALYS (Stacked) by label (total within the TARGET_PERIOD).
+ Throw error if not a record for every year in the TARGET PERIOD (to guard against inadvertently using
+ results from runs that crashed mid-way through the simulation.
+ """
+ years_needed = relevant_period_for_costing # [i.year for i in TARGET_PERIOD_INTERVENTION]
+ assert set(_df.year.unique()).issuperset(years_needed), "Some years are not recorded."
+ _df = _df.loc[_df.year.between(*years_needed)].drop(columns=['date', 'sex', 'age_range']).groupby('year').sum().sum(axis = 1)
+
+ # Initial year and discount rate
+ initial_year = min(_df.index.unique())
+
+ # Calculate the discounted values
+ discounted_values = _df / (1 + discount_rate) ** (_df.index - initial_year)
+
+ return pd.Series(discounted_values.sum())
+
+num_dalys = extract_results(
+ results_folder,
+ module='tlo.methods.healthburden',
+ key='dalys_stacked',
+ custom_generate_series=get_num_dalys,
+ do_scaling=True
+)
+
+# Get absolute DALYs averted
+num_dalys_averted = (-1.0 *
+ pd.DataFrame(
+ find_difference_relative_to_comparison(
+ num_dalys.loc[0],
+ comparison=0) # sets the comparator to 0 which is the Actual scenario
+ ).T.iloc[0].unstack(level='run'))
+num_dalys_averted_fcdo_scenarios = num_dalys_averted[
+ num_dalys_averted.index.get_level_values(0).isin(hss_scenarios_for_fcdo_report)]
+num_dalys_averted_gf_scenarios = num_dalys_averted[
+ num_dalys_averted.index.get_level_values(0).isin(hss_scenarios_for_gf_report)]
+
+# The monetary value of the health benefit is delta health times CET (negative values are set to 0)
+def get_monetary_value_of_incremental_health(_num_dalys_averted, _chosen_value_of_life_year):
+ monetary_value_of_incremental_health = (_num_dalys_averted * _chosen_value_of_life_year).clip(lower=0.0)
+ return monetary_value_of_incremental_health
+
+# TODO check that the above calculation is correct
+
+# 3. Return on Investment Plot
+# ----------------------------------------------------
+# FCDO
+# Combined ROI plot of relevant scenarios
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [1,2,3,4],
+ _scenario_dict = hss_scenarios,
+ _outputfilepath=roi_outputs_folder_fcdo,
+ _value_of_life_suffix = 'HR_VSL')
+
+# Combined ROI plot of relevant scenarios
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [6,7,8,9],
+ _scenario_dict = hss_scenarios,
+ _outputfilepath=roi_outputs_folder_fcdo,
+ _value_of_life_suffix = 'Consumables_VSL')
+
+# Combined ROI plot of relevant scenarios
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [10,12],
+ _scenario_dict = hss_scenarios,
+ _outputfilepath=roi_outputs_folder_fcdo,
+ _value_of_life_suffix = 'HSS_VSL')
+
+# Global Fund
+# Combined ROI plot of relevant scenarios
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [1,2,3,4,5],
+ _scenario_dict = hss_scenarios,
+ _outputfilepath=roi_outputs_folder_gf,
+ _value_of_life_suffix = 'HR_VSL')
+
+# Combined ROI plot of relevant scenarios
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [6,7,8],
+ _scenario_dict = hss_scenarios,
+ _outputfilepath=roi_outputs_folder_gf,
+ _value_of_life_suffix = 'Consumables_VSL')
+
+# Combined ROI plot of relevant scenarios
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [12],
+ _scenario_dict = hss_scenarios,
+ _outputfilepath=roi_outputs_folder_gf,
+ _value_of_life_suffix = 'HSS_VSL')
+
+# 4. Plot Maximum ability-to-pay at CET
+# ----------------------------------------------------
+max_ability_to_pay_for_implementation = (get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_cet) - incremental_scenario_cost).clip(
+ lower=0.0) # monetary value - change in costs
+max_ability_to_pay_for_implementation_summarized = summarize_cost_data(max_ability_to_pay_for_implementation)
+max_ability_to_pay_for_implementation_summarized_fcdo = max_ability_to_pay_for_implementation_summarized[
+ max_ability_to_pay_for_implementation_summarized.index.get_level_values(0).isin(hss_scenarios_for_fcdo_report)]
+max_ability_to_pay_for_implementation_summarized_gf = max_ability_to_pay_for_implementation_summarized[
+ max_ability_to_pay_for_implementation_summarized.index.get_level_values(0).isin(hss_scenarios_for_gf_report)]
+
+# FCDO
+# Plot Maximum ability to pay
+name_of_plot = f'Maximum ability to pay at CET, {relevant_period_for_costing[0]}-{relevant_period_for_costing[1]}'
+fig, ax = do_standard_bar_plot_with_ci(
+ (max_ability_to_pay_for_implementation_summarized_fcdo / 1e6),
+ annotations=[
+ f"{round(row['mean'] / 1e6, 1)} \n ({round(row['lower'] / 1e6, 1)}-\n {round(row['upper'] / 1e6, 1)})"
+ for _, row in max_ability_to_pay_for_implementation_summarized_fcdo.iterrows()
+ ],
+ xticklabels_horizontal_and_wrapped=False,
+)
+ax.set_title(name_of_plot)
+ax.set_ylabel('Maximum ability to pay \n(Millions)')
+fig.tight_layout()
+fig.savefig(roi_outputs_folder_fcdo / name_of_plot.replace(' ', '_').replace(',', ''))
+plt.close(fig)
+
+# Global Fund
+# Plot Maximum ability to pay
+name_of_plot = f'Maximum ability to pay at CET, {relevant_period_for_costing[0]}-{relevant_period_for_costing[1]}'
+fig, ax = do_standard_bar_plot_with_ci(
+ (max_ability_to_pay_for_implementation_summarized_gf / 1e6),
+ annotations=[
+ f"{round(row['mean'] / 1e6, 1)} \n ({round(row['lower'] / 1e6, 1)}-\n {round(row['upper'] / 1e6, 1)})"
+ for _, row in max_ability_to_pay_for_implementation_summarized_gf.iterrows()
+ ],
+ xticklabels_horizontal_and_wrapped=False,
+)
+ax.set_title(name_of_plot)
+ax.set_ylabel('Maximum ability to pay \n(Millions)')
+fig.tight_layout()
+fig.savefig(roi_outputs_folder_gf / name_of_plot.replace(' ', '_').replace(',', ''))
+plt.close(fig)
+
+# Plot incremental costs
+incremental_scenario_cost_summarized = summarize_cost_data(incremental_scenario_cost)
+# Keep only scenarios of interest
+incremental_scenario_cost_summarized_fcdo = incremental_scenario_cost_summarized[
+ incremental_scenario_cost_summarized.index.get_level_values(0).isin(hss_scenarios_for_fcdo_report)]
+incremental_scenario_cost_summarized_gf = incremental_scenario_cost_summarized[
+ incremental_scenario_cost_summarized.index.get_level_values(0).isin(hss_scenarios_for_gf_report)]
+
+# FCDO
+name_of_plot = f'Incremental scenario cost relative to baseline {relevant_period_for_costing[0]}-{relevant_period_for_costing[1]}'
+fig, ax = do_standard_bar_plot_with_ci(
+ (incremental_scenario_cost_summarized_fcdo / 1e6),
+ annotations=[
+ f"{round(row['mean'] / 1e6, 1)} \n ({round(row['lower'] / 1e6, 1)}- \n {round(row['upper'] / 1e6, 1)})"
+ for _, row in incremental_scenario_cost_summarized_fcdo.iterrows()
+ ],
+ xticklabels_horizontal_and_wrapped=False,
+)
+ax.set_title(name_of_plot)
+ax.set_ylabel('Cost \n(USD Millions)')
+fig.tight_layout()
+fig.savefig(roi_outputs_folder_fcdo / name_of_plot.replace(' ', '_').replace(',', ''))
+plt.close(fig)
+
+# Global Fund
+name_of_plot = f'Incremental scenario cost relative to baseline {relevant_period_for_costing[0]}-{relevant_period_for_costing[1]}'
+fig, ax = do_standard_bar_plot_with_ci(
+ (incremental_scenario_cost_summarized_gf / 1e6),
+ annotations=[
+ f"{round(row['mean'] / 1e6, 1)} \n ({round(row['lower'] / 1e6, 1)}- \n {round(row['upper'] / 1e6, 1)})"
+ for _, row in incremental_scenario_cost_summarized_gf.iterrows()
+ ],
+ xticklabels_horizontal_and_wrapped=False,
+)
+ax.set_title(name_of_plot)
+ax.set_ylabel('Cost \n(USD Millions)')
+fig.tight_layout()
+fig.savefig(roi_outputs_folder_gf / name_of_plot.replace(' ', '_').replace(',', ''))
+plt.close(fig)
+
+# 4. Plot costs
+# ----------------------------------------------------
+# FCDO
+input_costs_for_plot = input_costs[input_costs.draw.isin(hss_scenarios_for_fcdo_report)]
+# First summarize all input costs
+input_costs_for_plot_summarized = input_costs_for_plot.groupby(['draw', 'year', 'cost_subcategory', 'Facility_Level', 'cost_subgroup', 'cost_category']).agg(
+ mean=('cost', 'mean'),
+ lower=('cost', lambda x: x.quantile(0.025)),
+ upper=('cost', lambda x: x.quantile(0.975))
+).reset_index()
+input_costs_for_plot_summarized = input_costs_for_plot_summarized.melt(
+ id_vars=['draw', 'year', 'cost_subcategory', 'Facility_Level', 'cost_subgroup', 'cost_category'],
+ value_vars=['mean', 'lower', 'upper'],
+ var_name='stat',
+ value_name='cost'
+)
+
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'all', _disaggregate_by_subgroup = False, _outputfilepath = Path(figurespath / 'fcdo'), _scenario_dict = hs_scenarios_substitutedict_fcdo)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'all', _year = [2025], _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict_fcdo)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'human resources for health', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict_fcdo)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'medical consumables', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict_fcdo)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'medical equipment', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict_fcdo)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'other', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict_fcdo)
+
+# Global Fund
+input_costs_for_plot = input_costs[input_costs.draw.isin(hss_scenarios_for_gf_report)]
+# First summarize all input costs
+input_costs_for_plot_summarized = input_costs_for_plot.groupby(['draw', 'year', 'cost_subcategory', 'Facility_Level', 'cost_subgroup', 'cost_category']).agg(
+ mean=('cost', 'mean'),
+ lower=('cost', lambda x: x.quantile(0.025)),
+ upper=('cost', lambda x: x.quantile(0.975))
+).reset_index()
+input_costs_for_plot_summarized = input_costs_for_plot_summarized.melt(
+ id_vars=['draw', 'year', 'cost_subcategory', 'Facility_Level', 'cost_subgroup', 'cost_category'],
+ value_vars=['mean', 'lower', 'upper'],
+ var_name='stat',
+ value_name='cost'
+)
+
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'all', _disaggregate_by_subgroup = False, _outputfilepath = Path(figurespath / 'gf'), _scenario_dict = hs_scenarios_substitutedict_gf)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'all', _year = [2025], _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict_gf)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'human resources for health', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict_gf)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'medical consumables', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict_gf)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'medical equipment', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict_gf)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'other', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = hs_scenarios_substitutedict_gf)
diff --git a/src/scripts/costing/cost_analysis_htm_with_and_without_hss.py b/src/scripts/costing/cost_analysis_htm_with_and_without_hss.py
new file mode 100644
index 0000000000..01eb93e38d
--- /dev/null
+++ b/src/scripts/costing/cost_analysis_htm_with_and_without_hss.py
@@ -0,0 +1,604 @@
+import argparse
+from pathlib import Path
+from tlo import Date
+from collections import Counter, defaultdict
+
+import calendar
+import datetime
+import os
+import textwrap
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+from matplotlib.ticker import FuncFormatter
+import numpy as np
+import pandas as pd
+import ast
+import math
+
+from tlo.analysis.utils import (
+ extract_params,
+ extract_results,
+ get_scenario_info,
+ get_scenario_outputs,
+ load_pickled_dataframes,
+ make_age_grp_lookup,
+ make_age_grp_types,
+ summarize,
+ create_pickles_locally,
+ parse_log_file,
+ unflatten_flattened_multi_index_in_logging
+)
+
+from scripts.costing.cost_estimation import (estimate_input_cost_of_scenarios,
+ summarize_cost_data,
+ apply_discounting_to_cost_data,
+ do_stacked_bar_plot_of_cost_by_category,
+ do_line_plot_of_cost,
+ generate_roi_plots,
+ generate_multiple_scenarios_roi_plot)
+
+# Define a timestamp for script outputs
+timestamp = datetime.datetime.now().strftime("_%Y_%m_%d_%H_%M")
+
+# Print the start time of the script
+print('Script Start', datetime.datetime.now().strftime('%H:%M'))
+
+# Create folders to store results
+resourcefilepath = Path("./resources")
+outputfilepath = Path('./outputs/t.mangal@imperial.ac.uk')
+figurespath = Path('./outputs/global_fund_roi_analysis/htm_with_and_without_hss')
+if not os.path.exists(figurespath):
+ os.makedirs(figurespath)
+roi_outputs_folder = Path(figurespath / 'roi')
+if not os.path.exists(roi_outputs_folder):
+ os.makedirs(roi_outputs_folder)
+
+# Load result files
+#------------------------------------------------------------------------------------------------------------------
+results_folder = get_scenario_outputs('htm_with_and_without_hss-2024-10-22T163743Z.py', outputfilepath)[0]
+
+# Check can read results from draw=0, run=0
+log = load_pickled_dataframes(results_folder, 0, 0) # look at one log (so can decide what to extract)
+params = extract_params(results_folder)
+
+# Declare default parameters for cost analysis
+#------------------------------------------------------------------------------------------------------------------
+# Population scaling factor for malaria scale-up projections
+population_scaling_factor = log['tlo.methods.demography']['scaling_factor']['scaling_factor'].iloc[0]
+# Load the list of districts and their IDs
+district_dict = pd.read_csv(resourcefilepath / 'demography' / 'ResourceFile_Population_2010.csv')[
+ ['District_Num', 'District']].drop_duplicates()
+district_dict = dict(zip(district_dict['District_Num'], district_dict['District']))
+
+# Period relevant for costing
+TARGET_PERIOD_INTERVENTION = (Date(2025, 1, 1), Date(2035, 12, 31)) # This is the period that is costed
+relevant_period_for_costing = [i.year for i in TARGET_PERIOD_INTERVENTION]
+list_of_relevant_years_for_costing = list(range(relevant_period_for_costing[0], relevant_period_for_costing[1] + 1))
+
+# Scenarios
+htm_scenarios = {0:"Baseline", 1: "HSS PACKAGE: Perfect", 2: "HSS PACKAGE: Realistic", 3: "HIV Programs Scale-up WITHOUT HSS PACKAGE",
+4: "HIV Programs Scale-up WITH FULL HSS PACKAGE", 5: "HIV Programs Scale-up WITH REALISTIC HSS PACKAGE", 6: "TB Programs Scale-up WITHOUT HSS PACKAGE",
+7: "TB Programs Scale-up WITH FULL HSS PACKAGE", 8: "TB Programs Scale-up WITH REALISTIC HSS PACKAGE", 9: "Malaria Programs Scale-up WITHOUT HSS PACKAGE",
+10: "Malaria Programs Scale-up WITH FULL HSS PACKAGE", 11: "Malaria Programs Scale-up WITH REALISTIC HSS PACKAGE", 12: "HTM Programs Scale-up WITHOUT HSS PACKAGE",
+13: "HTM Programs Scale-up WITH FULL HSS PACKAGE", 14: "HTM Programs Scale-up WITH REALISTIC HSS PACKAGE", 15: "HTM Programs Scale-up WITH SUPPLY CHAINS", 16: "HTM Programs Scale-up WITH HRH"}
+
+htm_scenarios_substitutedict_fcdo = {0:"0", 1: "1", 2: "A", 3: "B",
+4: "4", 5: "C", 6: "D",
+7: "7", 8: "E", 9: "F",
+10: "10", 11: "G", 12: "H",
+13: "13", 14: "I", 15: "J", 16: "K"}
+
+# Subset of scenarios included in analysis
+htm_scenarios_for_gf_report = [0, 2, 3, 5, 6, 8, 9, 11, 12, 14, 15, 16]
+
+color_map = {
+ 'Baseline': '#9e0142',
+ 'HSS PACKAGE: Realistic': '#d8434e',
+ 'HIV Programs Scale-up WITHOUT HSS PACKAGE': '#f36b48',
+ 'HIV Programs Scale-up WITH REALISTIC HSS PACKAGE': '#fca45c',
+ 'TB Programs Scale-up WITHOUT HSS PACKAGE': '#fddc89',
+ 'TB Programs Scale-up WITH REALISTIC HSS PACKAGE': '#e7f7a0',
+ 'Malaria Programs Scale-up WITHOUT HSS PACKAGE': '#a5dc97',
+ 'Malaria Programs Scale-up WITH REALISTIC HSS PACKAGE': '#6dc0a6',
+ 'HTM Programs Scale-up WITHOUT HSS PACKAGE': '#438fba',
+ 'HTM Programs Scale-up WITH REALISTIC HSS PACKAGE': '#5e4fa2',
+ 'HTM Programs Scale-up WITH SUPPLY CHAINS': '#3c71aa',
+ 'HTM Programs Scale-up WITH HRH': '#2f6094',
+}
+
+# Cost-effectiveness threshold
+chosen_cet = 199.620811947318 # This is based on the estimate from Lomas et al (2023)- $160.595987085533 in 2019 USD coverted to 2023 USD
+# based on Ochalek et al (2018) - the paper provided the value $61 in 2016 USD terms, this value is $77.4 in 2023 USD terms
+chosen_value_of_statistical_life = 834
+
+# Discount rate
+discount_rate = 0.03
+
+# Define a function to create bar plots
+def do_bar_plot_with_ci(_df, annotations=None, xticklabels_horizontal_and_wrapped=False):
+ """Make a vertical bar plot for each row of _df, using the columns to identify the height of the bar and the
+ extent of the error bar."""
+
+ # Calculate y-error bars
+ yerr = np.array([
+ (_df['mean'] - _df['lower']).values,
+ (_df['upper'] - _df['mean']).values,
+ ])
+
+ # Map xticks based on the hss_scenarios dictionary
+ xticks = {index: htm_scenarios.get(index, f"Scenario {index}") for index in _df.index}
+
+ # Retrieve colors from color_map based on the xticks labels
+ colors = [color_map.get(label, '#333333') for label in xticks.values()] # default to grey if not found
+
+ # Generate consecutive x positions for the bars, ensuring no gaps
+ x_positions = np.arange(len(xticks)) # Consecutive integers for each bar position
+
+ fig, ax = plt.subplots()
+ ax.bar(
+ x_positions,
+ _df['mean'].values,
+ yerr=yerr,
+ color=colors, # Set bar colors
+ alpha=1,
+ ecolor='black',
+ capsize=10,
+ )
+
+ # Add optional annotations above each bar
+ if annotations:
+ for xpos, ypos, text in zip(x_positions, _df['upper'].values, annotations):
+ ax.text(xpos, ypos * 1.05, text, horizontalalignment='center', fontsize=8)
+
+ # Set x-tick labels with wrapped text if required
+ wrapped_labs = ["\n".join(textwrap.wrap(label,30)) for label in xticks.values()]
+ ax.set_xticks(x_positions) # Set x-ticks to consecutive positions
+ ax.set_xticklabels(wrapped_labs, rotation=45 if not xticklabels_horizontal_and_wrapped else 0, ha='right',
+ fontsize=7)
+
+ # Set y-axis limit to upper max + 500
+ ax.set_ylim(_df['lower'].min()*1.25, _df['upper'].max()*1.25)
+
+ # Set font size for y-tick labels and grid
+ ax.tick_params(axis='y', labelsize=9)
+ ax.tick_params(axis='x', labelsize=9)
+
+ ax.grid(axis="y")
+ ax.spines['top'].set_visible(False)
+ ax.spines['right'].set_visible(False)
+ fig.tight_layout()
+
+ return fig, ax
+
+def do_standard_bar_plot_with_ci(_df, set_colors=None, annotations=None,
+ xticklabels_horizontal_and_wrapped=False,
+ put_labels_in_legend=True,
+ offset=1e6):
+ """Make a vertical bar plot for each row of _df, using the columns to identify the height of the bar and the
+ extent of the error bar."""
+
+ substitute_labels = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+
+ yerr = np.array([
+ (_df['mean'] - _df['lower']).values,
+ (_df['upper'] - _df['mean']).values,
+ ])
+# TODO should be above be 'median'
+ xticks = {(i + 0.5): k for i, k in enumerate(_df.index)}
+
+ if set_colors:
+ colors = [color_map.get(series, 'grey') for series in _df.index]
+ else:
+ cmap = sns.color_palette('Spectral', as_cmap=True)
+ rescale = lambda y: (y - np.min(y)) / (np.max(y) - np.min(y)) # noqa: E731
+ colors = list(map(cmap, rescale(np.array(list(xticks.keys()))))) if put_labels_in_legend else None
+
+ fig, ax = plt.subplots(figsize=(10, 5))
+ ax.bar(
+ xticks.keys(),
+ _df['mean'].values,
+ yerr=yerr,
+ ecolor='black',
+ color=colors,
+ capsize=10,
+ label=xticks.values()
+ )
+
+ if annotations:
+ for xpos, (ypos, text) in zip(xticks.keys(), zip(_df['upper'].values.flatten(), annotations)):
+ annotation_y = ypos + offset
+
+ ax.text(
+ xpos,
+ annotation_y,
+ '\n'.join(text.split(' ', 1)),
+ horizontalalignment='center',
+ verticalalignment='bottom', # Aligns text at the bottom of the annotation position
+ fontsize='x-small',
+ rotation='horizontal'
+ )
+
+ ax.set_xticks(list(xticks.keys()))
+
+ if put_labels_in_legend:
+ # Update xticks label with substitute labels
+ # Insert legend with updated labels that shows correspondence between substitute label and original label
+ xtick_values = [letter for letter, label in zip(substitute_labels, xticks.values())]
+ xtick_legend = [f'{letter}: {label}' for letter, label in zip(substitute_labels, xticks.values())]
+ h, legs = ax.get_legend_handles_labels()
+ ax.legend(h, xtick_legend, loc='center left', fontsize='small', bbox_to_anchor=(1, 0.5))
+ ax.set_xticklabels(list(xtick_values))
+ else:
+ if not xticklabels_horizontal_and_wrapped:
+ # xticklabels will be vertical and not wrapped
+ ax.set_xticklabels(list(xticks.values()), rotation=90)
+ else:
+ wrapped_labs = ["\n".join(textwrap.wrap(_lab, 20)) for _lab in xticks.values()]
+ ax.set_xticklabels(wrapped_labs)
+
+ ax.grid(axis="y")
+ ax.spines['top'].set_visible(False)
+ ax.spines['right'].set_visible(False)
+ fig.tight_layout(pad=2.0)
+ plt.subplots_adjust(left=0.15, right=0.85) # Adjust left and right margins
+
+ return fig, ax
+
+# Estimate standard input costs of scenario
+#-----------------------------------------------------------------------------------------------------------------------
+input_costs = estimate_input_cost_of_scenarios(results_folder, resourcefilepath,
+ _years=list_of_relevant_years_for_costing, cost_only_used_staff=True,
+ _discount_rate = discount_rate)
+# _draws = htm_scenarios_for_gf_report --> this subset is created after calculating malaria scale up costs
+# TODO Remove the manual fix below once the logging for these is corrected
+input_costs.loc[input_costs.cost_subgroup == 'Oxygen, 1000 liters, primarily with oxygen cylinders', 'cost'] = \
+ input_costs.loc[input_costs.cost_subgroup == 'Oxygen, 1000 liters, primarily with oxygen cylinders', 'cost']/10
+input_costs.loc[input_costs.cost_subgroup == 'Depot-Medroxyprogesterone Acetate 150 mg - 3 monthly', 'cost'] =\
+ input_costs.loc[input_costs.cost_subgroup == 'Depot-Medroxyprogesterone Acetate 150 mg - 3 monthly', 'cost']/7
+#input_costs = apply_discounting_to_cost_data(input_costs, _discount_rate = discount_rate)
+
+# Add additional costs pertaining to simulation (Only for scenarios with Malaria scale-up)
+#-----------------------------------------------------------------------------------------------------------------------
+# Extract supply chain cost as a proportion of consumable costs to apply to malaria scale-up commodities
+# Load primary costing resourcefile
+workbook_cost = pd.read_excel((resourcefilepath / "costing/ResourceFile_Costing.xlsx"),
+ sheet_name=None)
+# Read parameters for consumables costs
+# Load consumables cost data
+unit_price_consumable = workbook_cost["consumables"]
+unit_price_consumable = unit_price_consumable.rename(columns=unit_price_consumable.iloc[0])
+unit_price_consumable = unit_price_consumable[['Item_Code', 'Final_price_per_chosen_unit (USD, 2023)']].reset_index(
+ drop=True).iloc[1:]
+unit_price_consumable = unit_price_consumable[unit_price_consumable['Item_Code'].notna()]
+
+# Assume that the cost of procurement, warehousing and distribution is a fixed proportion of consumable purchase costs
+# The fixed proportion is based on Resource Mapping Expenditure data from 2018
+resource_mapping_data = workbook_cost["resource_mapping_r7_summary"]
+# Make sure values are numeric
+expenditure_column = ['EXPENDITURE (USD) (Jul 2018 - Jun 2019)']
+resource_mapping_data[expenditure_column] = resource_mapping_data[expenditure_column].apply(
+ lambda x: pd.to_numeric(x, errors='coerce'))
+supply_chain_expenditure = \
+resource_mapping_data[resource_mapping_data['Cost Type'] == 'Supply Chain'][expenditure_column].sum()[0]
+consumables_purchase_expenditure = \
+resource_mapping_data[resource_mapping_data['Cost Type'] == 'Drugs and Commodities'][expenditure_column].sum()[0] + \
+resource_mapping_data[resource_mapping_data['Cost Type'] == 'HIV Drugs and Commodities'][expenditure_column].sum()[0]
+supply_chain_cost_proportion = supply_chain_expenditure / consumables_purchase_expenditure
+
+# In this case malaria intervention scale-up costs were not included in the standard estimate_input_cost_of_scenarios function
+list_of_draws_with_malaria_scaleup_parameters = params[(params.module_param == 'Malaria:scaleup_start_year')]
+list_of_draws_with_malaria_scaleup_parameters.loc[:,'value'] = pd.to_numeric(list_of_draws_with_malaria_scaleup_parameters['value'])
+list_of_draws_with_malaria_scaleup_implemented_in_costing_period = list_of_draws_with_malaria_scaleup_parameters[(list_of_draws_with_malaria_scaleup_parameters['value'] < max(relevant_period_for_costing))].index.to_list()
+
+# 1. IRS costs
+irs_coverage_rate = 0.8
+districts_with_irs_scaleup = ['Kasungu', 'Mchinji', 'Lilongwe', 'Lilongwe City', 'Dowa', 'Ntchisi', 'Salima', 'Mangochi',
+ 'Mwanza', 'Likoma', 'Nkhotakota']
+# Convert above list of district names to numeric district identifiers
+district_keys_with_irs_scaleup = [key for key, name in district_dict.items() if name in districts_with_irs_scaleup]
+TARGET_PERIOD_MALARIA_SCALEUP = (Date(2024, 1, 1), Date(2035, 12, 31))
+
+# Get population by district
+def get_total_population_by_district(_df):
+ years_needed = [i.year for i in TARGET_PERIOD_MALARIA_SCALEUP] # we only consider the population for the malaria scale-up period
+ # because those are the years relevant for malaria scale-up costing
+ _df['year'] = pd.to_datetime(_df['date']).dt.year
+ assert set(_df.year.unique()).issuperset(years_needed), "Some years are not recorded."
+ _df = pd.melt(_df.drop(columns = 'date'), id_vars = ['year']).rename(columns = {'variable': 'district'})
+ return pd.Series(
+ data=_df
+ .loc[_df.year.between(*years_needed)]
+ .set_index(['year', 'district'])['value']
+ )
+
+district_population_by_year = extract_results(
+ results_folder,
+ module='tlo.methods.malaria',
+ key='pop_district',
+ custom_generate_series=get_total_population_by_district,
+ do_scaling=True
+)
+
+def get_number_of_people_covered_by_malaria_scaleup(_df, list_of_districts_covered = None, draws_included = None):
+ _df = pd.DataFrame(_df)
+ # Reset the index to make 'district' a column
+ _df = _df.reset_index()
+ # Convert the 'district' column to numeric values
+ _df['district'] = pd.to_numeric(_df['district'], errors='coerce')
+ _df = _df.set_index(['year', 'district'])
+ if list_of_districts_covered is not None:
+ _df.loc[~_df.index.get_level_values('district').isin(list_of_districts_covered), :] = 0
+ if draws_included is not None:
+ _df.loc[:, ~_df.columns.get_level_values('draw').isin(draws_included)] = 0
+ return _df
+
+district_population_covered_by_irs_scaleup_by_year = get_number_of_people_covered_by_malaria_scaleup(district_population_by_year,
+ list_of_districts_covered=district_keys_with_irs_scaleup,
+ draws_included = list_of_draws_with_malaria_scaleup_implemented_in_costing_period)
+
+irs_cost_per_person = unit_price_consumable[unit_price_consumable.Item_Code == 161]['Final_price_per_chosen_unit (USD, 2023)']
+# The above unit cost already includes implementation - project management (17%), personnel (6%), vehicles (10%), equipment (6%), monitoring and evaluation (3%), training (3%),
+# other commodities (3%) and buildings (2%) from Alonso et al (2021)
+irs_multiplication_factor = irs_cost_per_person * irs_coverage_rate
+total_irs_cost = irs_multiplication_factor.iloc[0] * district_population_covered_by_irs_scaleup_by_year # for districts and scenarios included
+total_irs_cost = total_irs_cost.groupby(level='year').sum()
+
+# 2. Bednet costs
+bednet_coverage_rate = 0.7
+# We can assume 3-year lifespan of a bednet, each bednet covering 1.8 people.
+unit_cost_of_bednet = unit_price_consumable[unit_price_consumable.Item_Code == 160]['Final_price_per_chosen_unit (USD, 2023)'] * (1 + supply_chain_cost_proportion)
+# We add supply chain costs (procurement + distribution + warehousing) because the unit_cost does not include this
+annual_bednet_cost_per_person = unit_cost_of_bednet / 1.8 / 3
+bednet_multiplication_factor = bednet_coverage_rate * annual_bednet_cost_per_person
+
+district_population_covered_by_bednet_scaleup_by_year = get_number_of_people_covered_by_malaria_scaleup(district_population_by_year,
+ draws_included = list_of_draws_with_malaria_scaleup_implemented_in_costing_period) # All districts covered
+
+total_bednet_cost = bednet_multiplication_factor.iloc[0] * district_population_covered_by_bednet_scaleup_by_year # for scenarios included
+total_bednet_cost = total_bednet_cost.groupby(level='year').sum()
+
+# Malaria scale-up costs - TOTAL
+malaria_scaleup_costs = [
+ (total_irs_cost.reset_index(), 'cost_of_IRS_scaleup'),
+ (total_bednet_cost.reset_index(), 'cost_of_bednet_scaleup'),
+]
+def melt_and_label_malaria_scaleup_cost(_df, label):
+ multi_index = pd.MultiIndex.from_tuples(_df.columns)
+ _df.columns = multi_index
+
+ # reshape dataframe and assign 'draw' and 'run' as the correct column headers
+ melted_df = pd.melt(_df, id_vars=['year']).rename(columns={'variable_0': 'draw', 'variable_1': 'run'})
+ # Replace item_code with consumable_name_tlo
+ melted_df['cost_subcategory'] = label
+ melted_df['cost_category'] = 'other'
+ melted_df['cost_subgroup'] = 'NA'
+ melted_df['Facility_Level'] = 'all'
+ melted_df = melted_df.rename(columns={'value': 'cost'})
+ return melted_df
+
+# Iterate through additional costs, melt and concatenate
+for df, label in malaria_scaleup_costs:
+ new_df = melt_and_label_malaria_scaleup_cost(df, label)
+ input_costs = pd.concat([input_costs, new_df], ignore_index=True)
+
+# TODO Reduce the cost of Oxygen and Depo-medroxy temporarily which we figure out the issue with this
+# Extract input_costs for browsing
+input_costs.groupby(['draw', 'run', 'cost_category', 'cost_subcategory', 'cost_subgroup','year'])['cost'].sum().to_csv(figurespath / 'cost_detailed.csv')
+
+# %%
+# Return on Invesment analysis
+# Calculate incremental cost
+# -----------------------------------------------------------------------------------------------------------------------
+# Aggregate input costs for further analysis
+input_costs_subset = input_costs[
+ (input_costs['year'] >= relevant_period_for_costing[0]) & (input_costs['year'] <= relevant_period_for_costing[1])]
+# TODO the above step may not longer be needed
+total_input_cost = input_costs_subset.groupby(['draw', 'run'])['cost'].sum()
+total_input_cost_summarized = summarize_cost_data(total_input_cost.unstack(level='run'))
+def find_difference_relative_to_comparison(_ser: pd.Series,
+ comparison: str,
+ scaled: bool = False,
+ drop_comparison: bool = True,
+ ):
+ """Find the difference in the values in a pd.Series with a multi-index, between the draws (level 0)
+ within the runs (level 1), relative to where draw = `comparison`.
+ The comparison is `X - COMPARISON`."""
+ return _ser \
+ .unstack(level=0) \
+ .apply(lambda x: (x - x[comparison]) / (x[comparison] if scaled else 1.0), axis=1) \
+ .drop(columns=([comparison] if drop_comparison else [])) \
+ .stack()
+
+
+incremental_scenario_cost = (pd.DataFrame(
+ find_difference_relative_to_comparison(
+ total_input_cost,
+ comparison=0) # sets the comparator to 0 which is the Actual scenario
+).T.iloc[0].unstack()).T
+
+# Keep only scenarios of interest
+incremental_scenario_cost = incremental_scenario_cost[
+ incremental_scenario_cost.index.get_level_values(0).isin(htm_scenarios_for_gf_report)]
+
+# Monetary value of health impact
+# -----------------------------------------------------------------------------------------------------------------------
+def get_num_dalys(_df):
+ """Return total number of DALYS (Stacked) by label (total within the TARGET_PERIOD).
+ Throw error if not a record for every year in the TARGET PERIOD (to guard against inadvertently using
+ results from runs that crashed mid-way through the simulation.
+ """
+ years_needed = relevant_period_for_costing # [i.year for i in TARGET_PERIOD_INTERVENTION]
+ assert set(_df.year.unique()).issuperset(years_needed), "Some years are not recorded."
+ _df = _df.loc[_df.year.between(*years_needed)].drop(columns=['date', 'sex', 'age_range']).groupby('year').sum().sum(axis = 1)
+
+ # Initial year and discount rate
+ initial_year = min(_df.index.unique())
+
+ # Calculate the discounted values
+ discounted_values = _df / (1 + discount_rate) ** (_df.index - initial_year)
+
+ return pd.Series(discounted_values.sum())
+
+num_dalys = extract_results(
+ results_folder,
+ module='tlo.methods.healthburden',
+ key='dalys_stacked',
+ custom_generate_series=get_num_dalys,
+ do_scaling=True
+)
+
+# Get absolute DALYs averted
+num_dalys_averted = (-1.0 *
+ pd.DataFrame(
+ find_difference_relative_to_comparison(
+ num_dalys.loc[0],
+ comparison=0) # sets the comparator to 0 which is the Actual scenario
+ ).T.iloc[0].unstack(level='run'))
+num_dalys_averted = num_dalys_averted[num_dalys_averted.index.get_level_values(0).isin(htm_scenarios_for_gf_report)]
+
+# The monetary value of the health benefit is delta health times CET (negative values are set to 0)
+def get_monetary_value_of_incremental_health(_num_dalys_averted, _chosen_value_of_life_year):
+ monetary_value_of_incremental_health = (_num_dalys_averted * _chosen_value_of_life_year).clip(lower=0.0)
+ return monetary_value_of_incremental_health
+
+# TODO check that the above calculation is correct
+
+# 3. Return on Investment Plot
+# ----------------------------------------------------
+# Plot ROI at various levels of cost
+generate_roi_plots(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_cet),
+ _incremental_input_cost=incremental_scenario_cost,
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _value_of_life_suffix = 'CET')
+
+generate_roi_plots(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _value_of_life_suffix = 'VSL')
+
+# Combined ROI plot of relevant scenarios
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [2,12,14],
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _value_of_life_suffix = 'VSL')
+
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [2,12,14,15,16],
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _value_of_life_suffix = 'all_HTM_VSL')
+
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [3,5],
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _value_of_life_suffix = 'HIV_VSL')
+
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [6,8],
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _value_of_life_suffix = 'TB_VSL')
+
+generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health=get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life),
+ _incremental_input_cost=incremental_scenario_cost,
+ _draws = [9,11],
+ _scenario_dict = htm_scenarios,
+ _outputfilepath=roi_outputs_folder,
+ _value_of_life_suffix = 'Malaria_VSL')
+
+# 4. Plot Maximum ability-to-pay at CET
+# ----------------------------------------------------
+max_ability_to_pay_for_implementation = (get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_cet) - incremental_scenario_cost).clip(
+ lower=0.0) # monetary value - change in costs
+max_ability_to_pay_for_implementation_summarized = summarize_cost_data(max_ability_to_pay_for_implementation)
+max_ability_to_pay_for_implementation_summarized = max_ability_to_pay_for_implementation_summarized[
+ max_ability_to_pay_for_implementation_summarized.index.get_level_values(0).isin(htm_scenarios_for_gf_report)]
+
+# Plot Maximum ability to pay
+name_of_plot = f'Maximum ability to pay at CET, {relevant_period_for_costing[0]}-{relevant_period_for_costing[1]}'
+fig, ax = do_standard_bar_plot_with_ci(
+ (max_ability_to_pay_for_implementation_summarized / 1e6),
+ annotations=[
+ f"{round(row['mean'] / 1e6, 1)} \n ({round(row['lower'] / 1e6, 1)}-\n {round(row['upper'] / 1e6, 1)})"
+ for _, row in max_ability_to_pay_for_implementation_summarized.iterrows()
+ ],
+ xticklabels_horizontal_and_wrapped=False,
+)
+ax.set_title(name_of_plot)
+ax.set_ylabel('Maximum ability to pay \n(Millions)')
+fig.tight_layout()
+fig.savefig(roi_outputs_folder / name_of_plot.replace(' ', '_').replace(',', ''))
+plt.close(fig)
+
+# 4. Plot Maximum ability-to-pay at VSL
+# ----------------------------------------------------
+max_ability_to_pay_for_implementation = (get_monetary_value_of_incremental_health(num_dalys_averted, _chosen_value_of_life_year = chosen_value_of_statistical_life) - incremental_scenario_cost).clip(
+ lower=0.0) # monetary value - change in costs
+max_ability_to_pay_for_implementation_summarized = summarize_cost_data(max_ability_to_pay_for_implementation)
+max_ability_to_pay_for_implementation_summarized = max_ability_to_pay_for_implementation_summarized[
+ max_ability_to_pay_for_implementation_summarized.index.get_level_values(0).isin(htm_scenarios_for_gf_report)]
+
+# Plot Maximum ability to pay
+name_of_plot = f'Maximum ability to pay at VSL, {relevant_period_for_costing[0]}-{relevant_period_for_costing[1]}'
+fig, ax = do_bar_plot_with_ci(
+ (max_ability_to_pay_for_implementation_summarized / 1e6),
+ annotations=[
+ f"{round(row['mean'] / 1e6, 1)} \n ({round(row['lower'] / 1e6, 1)}-\n {round(row['upper'] / 1e6, 1)})"
+ for _, row in max_ability_to_pay_for_implementation_summarized.iterrows()
+ ],
+ xticklabels_horizontal_and_wrapped=False,
+)
+ax.set_title(name_of_plot)
+ax.set_ylabel('Maximum ability to pay (at VSL) \n(Millions)')
+fig.tight_layout()
+fig.savefig(roi_outputs_folder / name_of_plot.replace(' ', '_').replace(',', ''))
+plt.close(fig)
+
+# Plot incremental costs
+incremental_scenario_cost_summarized = summarize_cost_data(incremental_scenario_cost)
+name_of_plot = f'Incremental scenario cost relative to baseline {relevant_period_for_costing[0]}-{relevant_period_for_costing[1]}'
+fig, ax = do_standard_bar_plot_with_ci(
+ (incremental_scenario_cost_summarized / 1e6),
+ annotations=[
+ f"{round(row['mean'] / 1e6, 1)} \n ({round(row['lower'] / 1e6, 1)}- \n {round(row['upper'] / 1e6, 1)})"
+ for _, row in incremental_scenario_cost_summarized.iterrows()
+ ],
+ xticklabels_horizontal_and_wrapped=False,
+)
+ax.set_title(name_of_plot)
+ax.set_ylabel('Cost \n(USD Millions)')
+fig.tight_layout()
+fig.savefig(roi_outputs_folder / name_of_plot.replace(' ', '_').replace(',', ''))
+plt.close(fig)
+
+# 4. Plot costs
+# ----------------------------------------------------
+input_costs_for_plot = input_costs[input_costs.draw.isin(htm_scenarios_for_gf_report)]
+# First summarize all input costs
+input_costs_for_plot_summarized = input_costs_for_plot.groupby(['draw', 'year', 'cost_subcategory', 'Facility_Level', 'cost_subgroup', 'cost_category']).agg(
+ mean=('cost', 'mean'),
+ lower=('cost', lambda x: x.quantile(0.025)),
+ upper=('cost', lambda x: x.quantile(0.975))
+).reset_index()
+input_costs_for_plot_summarized = input_costs_for_plot_summarized.melt(
+ id_vars=['draw', 'year', 'cost_subcategory', 'Facility_Level', 'cost_subgroup', 'cost_category'],
+ value_vars=['mean', 'lower', 'upper'],
+ var_name='stat',
+ value_name='cost'
+)
+
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'all', _year = list(range(2025, 2036)), _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = htm_scenarios_substitutedict)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'all', _year = [2025], _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = htm_scenarios)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'human resources for health', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = htm_scenarios)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'medical consumables', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = htm_scenarios)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'medical equipment', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = htm_scenarios)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_for_plot_summarized, _cost_category = 'other', _disaggregate_by_subgroup = False, _outputfilepath = figurespath, _scenario_dict = htm_scenarios)
diff --git a/src/scripts/costing/cost_estimation.py b/src/scripts/costing/cost_estimation.py
new file mode 100644
index 0000000000..4168e5ed12
--- /dev/null
+++ b/src/scripts/costing/cost_estimation.py
@@ -0,0 +1,1947 @@
+import argparse
+from pathlib import Path
+from tlo import Date
+from collections import Counter, defaultdict
+
+import calendar
+import datetime
+import os
+import textwrap
+
+import matplotlib.pyplot as plt
+from matplotlib.ticker import FuncFormatter
+import squarify
+import numpy as np
+import pandas as pd
+import ast
+import math
+import itertools
+from itertools import cycle
+
+from tlo.analysis.utils import (
+ extract_params,
+ extract_results,
+ get_scenario_info,
+ get_scenario_outputs,
+ load_pickled_dataframes,
+ make_age_grp_lookup,
+ make_age_grp_types,
+ summarize,
+ create_pickles_locally,
+ parse_log_file,
+ unflatten_flattened_multi_index_in_logging
+)
+
+# Define a timestamp for script outputs
+timestamp = datetime.datetime.now().strftime("_%Y_%m_%d_%H_%M")
+
+# Print the start time of the script
+print('Script Start', datetime.datetime.now().strftime('%H:%M'))
+
+#%%
+
+# Define a function to discount and summarise costs by cost_category
+def apply_discounting_to_cost_data(_df, _discount_rate=0, _year = None):
+ if _year == None:
+ # Initial year and discount rate
+ initial_year = min(_df['year'].unique())
+ else:
+ initial_year = _year
+
+ # Calculate the discounted values
+ _df.loc[:, 'cost'] = _df['cost'] / ((1 + _discount_rate) ** (_df['year'] - initial_year))
+ return _df
+
+def estimate_input_cost_of_scenarios(results_folder: Path,
+ resourcefilepath: Path = None,
+ _draws = None, _runs = None,
+ summarize: bool = False, _years = None,
+ cost_only_used_staff: bool = True,
+ _discount_rate = 0):
+ # Useful common functions
+ def drop_outside_period(_df):
+ """Return a dataframe which only includes for which the date is within the limits defined by TARGET_PERIOD"""
+ return _df.drop(index=_df.index[~_df['date'].between(*TARGET_PERIOD)])
+
+ def melt_model_output_draws_and_runs(_df, id_vars):
+ multi_index = pd.MultiIndex.from_tuples(_df.columns)
+ _df.columns = multi_index
+ melted_df = pd.melt(_df, id_vars=id_vars).rename(columns={'variable_0': 'draw', 'variable_1': 'run'})
+ return melted_df
+
+ # Define a relative pathway for relavant folders
+ path_for_consumable_resourcefiles = resourcefilepath / "healthsystem/consumables"
+
+ # %% Gathering basic information
+ # Load basic simulation parameters
+ #-------------------------------------
+ log = load_pickled_dataframes(results_folder, 0, 0) # read from 1 draw and run
+ info = get_scenario_info(results_folder) # get basic information about the results
+ if _draws is None:
+ _draws = range(0, info['number_of_draws'])
+ if _runs is None:
+ _runs = range(0, info['runs_per_draw'])
+ final_year_of_simulation = max(log['tlo.methods.healthsystem.summary']['hsi_event_counts']['date']).year
+ first_year_of_simulation = min(log['tlo.methods.healthsystem.summary']['hsi_event_counts']['date']).year
+ years = list(range(first_year_of_simulation, final_year_of_simulation + 1))
+
+ # Load cost input files
+ #------------------------
+ # Load primary costing resourcefile
+ workbook_cost = pd.read_excel((resourcefilepath / "costing/ResourceFile_Costing.xlsx"),
+ sheet_name = None)
+
+ # Extract districts and facility levels from the Master Facility List
+ mfl = pd.read_csv(resourcefilepath / "healthsystem" / "organisation" / "ResourceFile_Master_Facilities_List.csv")
+ district_dict = pd.read_csv(resourcefilepath / 'demography' / 'ResourceFile_Population_2010.csv')[['District_Num', 'District']].drop_duplicates()
+ district_dict = dict(zip(district_dict['District_Num'], district_dict['District']))
+ facility_id_levels_dict = dict(zip(mfl['Facility_ID'], mfl['Facility_Level']))
+ fac_levels = set(mfl.Facility_Level)
+
+ # Overall cost assumptions
+ TARGET_PERIOD = (Date(first_year_of_simulation, 1, 1), Date(final_year_of_simulation, 12, 31)) # Declare period for which the results will be generated (defined inclusively)
+ discount_rate = 0.03 # this is the discount rate for annuitization
+
+ # Read all cost parameters
+ #---------------------------------------
+ # Read parameters for HR costs
+ hr_cost_parameters = workbook_cost["human_resources"]
+ hr_cost_parameters['Facility_Level'] = hr_cost_parameters['Facility_Level'].astype(str) # Store Facility_Level as string
+
+ # Read parameters for consumables costs
+ # Load consumables cost data
+ unit_price_consumable = workbook_cost["consumables"]
+ unit_price_consumable = unit_price_consumable.rename(columns=unit_price_consumable.iloc[0])
+ unit_price_consumable = unit_price_consumable[['Item_Code', 'Final_price_per_chosen_unit (USD, 2023)']].reset_index(drop=True).iloc[1:]
+ unit_price_consumable = unit_price_consumable[unit_price_consumable['Item_Code'].notna()]
+
+ # Load and prepare equipment cost parameters
+ # Unit costs of equipment
+ unit_cost_equipment = workbook_cost["equipment"]
+ unit_cost_equipment = unit_cost_equipment.rename(columns=unit_cost_equipment.iloc[7]).reset_index(drop=True).iloc[8:]
+ unit_cost_equipment = unit_cost_equipment[unit_cost_equipment['Item_code'].notna()] # drop empty row
+ # Calculate necessary costs based on HSSP-III assumptions
+ unit_cost_equipment['replacement_cost_annual'] = unit_cost_equipment.apply(lambda row: row['unit_purchase_cost']/(1+(1-(1+discount_rate)**(-row['Life span']+1))/discount_rate), axis=1) # Annuitised over the life span of the equipment assuming outlay at the beginning of the year
+ unit_cost_equipment['service_fee_annual'] = unit_cost_equipment.apply(lambda row: row['unit_purchase_cost'] * 0.8 / 8 if row['unit_purchase_cost'] > 1000 else 0, axis=1) # 80% of the value of the item over 8 years
+ unit_cost_equipment['spare_parts_annual'] = unit_cost_equipment.apply(lambda row: row['unit_purchase_cost'] * 0.2 / 8 if row['unit_purchase_cost'] > 1000 else 0, axis=1) # 20% of the value of the item over 8 years
+ unit_cost_equipment['major_corrective_maintenance_cost_annual'] = unit_cost_equipment.apply(lambda row: row['unit_purchase_cost'] * 0.2 * 0.2 / 8 if row['unit_purchase_cost'] < 250000 else 0, axis=1) # 20% of the value of 20% of the items over 8 years
+ # TODO consider discounting the other components
+ # Quantity needed for each equipment by facility
+ unit_cost_equipment = unit_cost_equipment[['Item_code','Equipment_tlo',
+ 'replacement_cost_annual', 'service_fee_annual', 'spare_parts_annual', 'major_corrective_maintenance_cost_annual',
+ 'Health Post_prioritised', 'Community_prioritised', 'Health Center_prioritised', 'District_prioritised', 'Central_prioritised']]
+ unit_cost_equipment = unit_cost_equipment.rename(columns={col: 'Quantity_' + col.replace('_prioritised', '') for col in unit_cost_equipment.columns if col.endswith('_prioritised')})
+ unit_cost_equipment = unit_cost_equipment.rename(columns={col: col.replace(' ', '_') for col in unit_cost_equipment.columns})
+
+ unit_cost_equipment = pd.wide_to_long(unit_cost_equipment, stubnames=['Quantity_'],
+ i=['Item_code', 'Equipment_tlo', 'replacement_cost_annual', 'service_fee_annual', 'spare_parts_annual', 'major_corrective_maintenance_cost_annual'],
+ j='Facility_Level', suffix='(\d+|\w+)').reset_index()
+ facility_level_mapping = {'Health_Post': '0', 'Health_Center': '1a', 'Community': '1b', 'District': '2', 'Central': '3'}
+ unit_cost_equipment['Facility_Level'] = unit_cost_equipment['Facility_Level'].replace(facility_level_mapping)
+ unit_cost_equipment = unit_cost_equipment.rename(columns = {'Quantity_': 'Quantity'})
+
+ # Load and prepare facility operation cost parameters
+ unit_cost_fac_operations = workbook_cost["facility_operations"]
+
+ # Function to prepare cost dataframe ready to be merged across cross categories
+ def retain_relevant_column_subset(_df, _category_specific_group):
+ columns_to_retain = ['draw', 'run', 'year', 'cost_subcategory', 'Facility_Level', _category_specific_group, 'cost']
+ if 'cost_category' in _df.columns:
+ columns_to_retain.append('cost_category')
+ _df = _df[columns_to_retain]
+ return _df
+ def prepare_cost_dataframe(_df, _category_specific_group, _cost_category):
+ _df = _df.rename(columns = {_category_specific_group: 'cost_subgroup'})
+ _df['cost_category'] = _cost_category
+ return retain_relevant_column_subset(_df, 'cost_subgroup')
+
+
+ # CALCULATE ECONOMIC COSTS
+ #%%
+ # 1. HR cost
+ #------------------------
+ print("Now estimating HR costs...")
+ # Define a function to merge unit cost data with model outputs
+ def merge_cost_and_model_data(cost_df, model_df, varnames):
+ merged_df = model_df.copy()
+ for varname in varnames:
+ new_cost_df = cost_df[cost_df['Parameter_name'] == varname][['OfficerType', 'Facility_Level', 'Value']]
+ new_cost_df = new_cost_df.rename(columns={"Value": varname})
+ if ((new_cost_df['OfficerType'] == 'All').all()) and ((new_cost_df['Facility_Level'] == 'All').all()):
+ merged_df[varname] = new_cost_df[varname].mean()
+ elif ((new_cost_df['OfficerType'] == 'All').all()) and ((new_cost_df['Facility_Level'] == 'All').all() == False):
+ merged_df = pd.merge(merged_df, new_cost_df[['Facility_Level',varname]], on=['Facility_Level'], how="left")
+ elif ((new_cost_df['OfficerType'] == 'All').all() == False) and ((new_cost_df['Facility_Level'] == 'All').all()):
+ merged_df = pd.merge(merged_df, new_cost_df[['OfficerType',varname]], on=['OfficerType'], how="left")
+ else:
+ merged_df = pd.merge(merged_df, new_cost_df, on=['OfficerType', 'Facility_Level'], how="left")
+ return merged_df
+
+ # Get available staff count for each year and draw
+ def get_staff_count_by_facid_and_officer_type(_df: pd.Series) -> pd.Series:
+ """Summarise the parsed logged-key results for one draw (as dataframe) into a pd.Series."""
+ _df = _df.set_axis(_df['date'].dt.year).drop(columns=['date'])
+ _df.index.name = 'year'
+
+ def change_to_standard_flattened_index_format(col):
+ parts = col.split("_", 3) # Split by "_" only up to 3 parts
+ if len(parts) > 2:
+ return parts[0] + "=" + parts[1] + "|" + parts[2] + "=" + parts[3] # Rejoin with "I" at the second occurrence
+ return col # If there's no second underscore, return the string as it is
+ _df.columns = [change_to_standard_flattened_index_format(col) for col in _df.columns]
+
+ return unflatten_flattened_multi_index_in_logging(_df).stack(level=[0, 1]) # expanded flattened axis
+
+ # Staff count by Facility ID
+ available_staff_count_by_facid_and_officertype = extract_results(
+ Path(results_folder),
+ module='tlo.methods.healthsystem.summary',
+ key='number_of_hcw_staff',
+ custom_generate_series=get_staff_count_by_facid_and_officer_type,
+ do_scaling=True,
+ )
+
+ # Update above series to get staff count by Facility_Level
+ available_staff_count_by_facid_and_officertype = available_staff_count_by_facid_and_officertype.reset_index().rename(columns= {'FacilityID': 'Facility_ID', 'Officer': 'OfficerType'})
+ available_staff_count_by_facid_and_officertype['Facility_ID'] = pd.to_numeric(available_staff_count_by_facid_and_officertype['Facility_ID'])
+ available_staff_count_by_facid_and_officertype['Facility_Level'] = available_staff_count_by_facid_and_officertype['Facility_ID'].map(facility_id_levels_dict)
+ idx = pd.IndexSlice
+ available_staff_count_by_level_and_officer_type = available_staff_count_by_facid_and_officertype.drop(columns = [idx['Facility_ID']]).groupby([idx['year'], idx['Facility_Level'], idx['OfficerType']]).sum()
+ available_staff_count_by_level_and_officer_type = melt_model_output_draws_and_runs(available_staff_count_by_level_and_officer_type.reset_index(), id_vars= ['year', 'Facility_Level', 'OfficerType'])
+ available_staff_count_by_level_and_officer_type['Facility_Level'] = available_staff_count_by_level_and_officer_type['Facility_Level'].astype(str) # make sure facility level is stored as string
+ available_staff_count_by_level_and_officer_type = available_staff_count_by_level_and_officer_type.drop(available_staff_count_by_level_and_officer_type[available_staff_count_by_level_and_officer_type['Facility_Level'] == '5'].index) # drop headquarters because we're only concerned with staff engaged in service delivery
+ available_staff_count_by_level_and_officer_type.rename(columns ={'value': 'staff_count'}, inplace=True)
+
+ # Get list of cadres which were utilised in each run to get the count of staff used in the simulation
+ # Note that we still cost the full staff count for any cadre-Facility_Level combination that was ever used in a run, and
+ # not the amount of time which was used
+ def get_capacity_used_by_officer_type_and_facility_level(_df: pd.Series) -> pd.Series:
+ """Summarise the parsed logged-key results for one draw (as dataframe) into a pd.Series."""
+ _df = _df.set_axis(_df['date'].dt.year).drop(columns=['date'])
+ _df.index.name = 'year'
+ return unflatten_flattened_multi_index_in_logging(_df).stack(level=[0, 1]) # expanded flattened axis
+
+ annual_capacity_used_by_cadre_and_level = extract_results(
+ Path(results_folder),
+ module='tlo.methods.healthsystem.summary',
+ key='Capacity_By_OfficerType_And_FacilityLevel',
+ custom_generate_series=get_capacity_used_by_officer_type_and_facility_level,
+ do_scaling=False,
+ )
+
+ # Prepare capacity used dataframe to be multiplied by staff count
+ average_capacity_used_by_cadre_and_level = annual_capacity_used_by_cadre_and_level.groupby(['OfficerType', 'FacilityLevel']).mean().reset_index(drop=False)
+ # TODO see if cadre-level combinations should be chosen by year
+ average_capacity_used_by_cadre_and_level.reset_index(drop=True) # Flatten multi=index column
+ average_capacity_used_by_cadre_and_level = average_capacity_used_by_cadre_and_level.melt(id_vars=['OfficerType', 'FacilityLevel'],
+ var_name=['draw', 'run'],
+ value_name='capacity_used')
+ list_of_cadre_and_level_combinations_used = average_capacity_used_by_cadre_and_level[average_capacity_used_by_cadre_and_level['capacity_used'] != 0][['OfficerType', 'FacilityLevel', 'draw', 'run']]
+ print(f"Out of {average_capacity_used_by_cadre_and_level.groupby(['OfficerType', 'FacilityLevel']).size().count()} cadre and level combinations available, {list_of_cadre_and_level_combinations_used.groupby(['OfficerType', 'FacilityLevel']).size().count()} are used across the simulations")
+ list_of_cadre_and_level_combinations_used = list_of_cadre_and_level_combinations_used.rename(columns = {'FacilityLevel':'Facility_Level'})
+
+ # Subset scenario staffing level to only include cadre-level combinations used in the simulation
+ used_staff_count_by_level_and_officer_type = available_staff_count_by_level_and_officer_type.merge(list_of_cadre_and_level_combinations_used, on = ['draw','run','OfficerType', 'Facility_Level'], how = 'right', validate = 'm:m')
+ used_staff_count_by_level_and_officer_type.rename(columns ={'value': 'staff_count'}, inplace=True)
+
+ if (cost_only_used_staff):
+ print("The input for 'cost_only_used_staff' implies that only cadre-level combinations which have been used in the run are costed")
+ staff_size_chosen_for_costing = used_staff_count_by_level_and_officer_type
+ else:
+ print("The input for 'cost_only_used_staff' implies that all staff are costed regardless of the cadre-level combinations which have been used in the run are costed")
+ staff_size_chosen_for_costing = available_staff_count_by_level_and_officer_type
+
+ # Calculate various components of HR cost
+ # 1.1 Salary cost for health workforce cadres used in the simulation (Staff count X Annual salary)
+ #---------------------------------------------------------------------------------------------------------------
+ salary_for_staff = merge_cost_and_model_data(cost_df = hr_cost_parameters, model_df = staff_size_chosen_for_costing,
+ varnames = ['salary_usd'])
+ salary_for_staff['cost'] = salary_for_staff['salary_usd'] * salary_for_staff['staff_count']
+
+ # 1.2 Pre-service training & recruitment cost to fill gap created by attrition
+ #---------------------------------------------------------------------------------------------------------------
+ preservice_training_cost = merge_cost_and_model_data(cost_df = hr_cost_parameters, model_df = staff_size_chosen_for_costing,
+ varnames = ['annual_attrition_rate',
+ 'licensure_exam_passing_rate', 'graduation_rate',
+ 'absorption_rate_of_students_into_public_workforce', 'proportion_of_workforce_recruited_from_abroad',
+ 'average_annual_preservice_training_cost_for_cadre', 'preservice_training_duration', 'recruitment_cost_per_person_recruited_usd',
+ 'average_length_of_tenure_in_the_public_sector'])
+
+ def calculate_npv_past_training_expenses_by_row(row, r = discount_rate):
+ # Initialize the NPV for the row
+ npv = 0
+ annual_cost = row['average_annual_preservice_training_cost_for_cadre']
+ full_years = int(row['preservice_training_duration']) # Extract integer part of the year
+ partial_year = row['preservice_training_duration'] - full_years # Fractional part of the year
+
+ # Iterate over each year of the training duration to calculate compounded cost to the present
+ # Calculate NPV for each full year of training
+ for t in range(full_years):
+ npv += annual_cost * (1 + r) ** (t+1+1) # 1 added twice because range(4) is [0,1,2,3]
+
+ # Account for the fractional year at the end if it exists
+ if partial_year > 0:
+ npv += annual_cost * partial_year * (1 + r) ** (1+r)
+
+ # Add recruitment cost assuming this happens during the partial year or the year after graduation if partial year == 0
+ npv += row['recruitment_cost_per_person_recruited_usd'] * (1+r)
+
+ return npv
+
+ # Calculate NPV for each row using iterrows and store in a new column
+ npv_values = []
+ for index, row in preservice_training_cost.iterrows():
+ npv = calculate_npv_past_training_expenses_by_row(row, r=discount_rate)
+ npv_values.append(npv)
+
+ preservice_training_cost['npv_of_training_and_recruitment_cost'] = npv_values
+ preservice_training_cost['npv_of_training_and_recruitment_cost_per_recruit'] = preservice_training_cost['npv_of_training_and_recruitment_cost'] *\
+ (1/(preservice_training_cost['absorption_rate_of_students_into_public_workforce'] + preservice_training_cost['proportion_of_workforce_recruited_from_abroad'])) *\
+ (1/preservice_training_cost['graduation_rate']) * (1/preservice_training_cost['licensure_exam_passing_rate'])
+ preservice_training_cost['annuitisation_rate'] = 1 + (1 - (1 + discount_rate) ** (-preservice_training_cost['average_length_of_tenure_in_the_public_sector'] + 1)) / discount_rate
+ preservice_training_cost['annuitised_training_and_recruitment_cost_per_recruit'] = preservice_training_cost['npv_of_training_and_recruitment_cost_per_recruit']/preservice_training_cost['annuitisation_rate']
+
+ # Cost per student trained * 1/Rate of absorption from the local and foreign graduates * 1/Graduation rate * attrition rate
+ # the inverse of attrition rate is the average expected tenure; and the preservice training cost needs to be divided by the average tenure
+ preservice_training_cost['cost'] = preservice_training_cost['annuitised_training_and_recruitment_cost_per_recruit'] * preservice_training_cost['staff_count'] * preservice_training_cost['annual_attrition_rate'] # not multiplied with attrition rate again because this is already factored into 'Annual_cost_per_staff_recruited'
+ preservice_training_cost = preservice_training_cost[['draw', 'run', 'year', 'OfficerType', 'Facility_Level', 'cost']]
+
+ # 1.3 In-service training cost to train all staff
+ #---------------------------------------------------------------------------------------------------------------
+ inservice_training_cost = merge_cost_and_model_data(cost_df = hr_cost_parameters, model_df = staff_size_chosen_for_costing,
+ varnames = ['annual_inservice_training_cost_usd'])
+ inservice_training_cost['cost'] = inservice_training_cost['staff_count'] * inservice_training_cost['annual_inservice_training_cost_usd']
+ inservice_training_cost = inservice_training_cost[['draw', 'run', 'year', 'OfficerType', 'Facility_Level', 'cost']]
+ # TODO Consider calculating economic cost of HR by multiplying salary times staff count with cadres_utilisation_rate
+
+ # 1.4 Regular mentorship and supportive supervision costs
+ #---------------------------------------------------------------------------------------------------------------
+ mentorship_and_supportive_cost = merge_cost_and_model_data(cost_df = hr_cost_parameters, model_df = staff_size_chosen_for_costing,
+ varnames = ['annual_mentorship_and_supervision_cost'])
+ mentorship_and_supportive_cost['cost'] = mentorship_and_supportive_cost['staff_count'] * mentorship_and_supportive_cost['annual_mentorship_and_supervision_cost']
+ mentorship_and_supportive_cost = mentorship_and_supportive_cost[['draw', 'run', 'year', 'OfficerType', 'Facility_Level', 'cost']]
+ # TODO Consider calculating economic cost of HR by multiplying salary times staff count with cadres_utilisation_rate
+
+ # 1.5 Store all HR costs in one standard format dataframe
+ #---------------------------------------------------------------------------------------------------------------
+ # Function to melt and label the cost category
+ def label_rows_of_cost_dataframe(_df, label_var, label):
+ _df = _df.reset_index()
+ _df[label_var] = label
+ return _df
+
+ # Initialize HR with the salary data
+ if (cost_only_used_staff):
+ human_resource_costs = retain_relevant_column_subset(label_rows_of_cost_dataframe(salary_for_staff, 'cost_subcategory', 'salary_for_cadres_used'), 'OfficerType')
+ # Concatenate additional cost categories
+ additional_costs = [
+ (preservice_training_cost, 'preservice_training_and_recruitment_cost_for_attrited_workers'),
+ (inservice_training_cost, 'inservice_training_cost_for_cadres_used'),
+ (mentorship_and_supportive_cost, 'mentorship_and_supportive_cost_for_cadres_used')
+ ]
+ else:
+ human_resource_costs = retain_relevant_column_subset(label_rows_of_cost_dataframe(salary_for_staff, 'cost_subcategory', 'salary_for_all_staff'), 'OfficerType')
+ # Concatenate additional cost categories
+ additional_costs = [
+ (preservice_training_cost, 'preservice_training_and_recruitment_cost_for_attrited_workers'),
+ (inservice_training_cost, 'inservice_training_cost_for_all_staff'),
+ (mentorship_and_supportive_cost, 'mentorship_and_supportive_cost_for_all_staff')
+ ]
+
+ # Iterate through additional costs, melt and concatenate
+ for df, label in additional_costs:
+ labelled_df = retain_relevant_column_subset(label_rows_of_cost_dataframe(df, 'cost_subcategory', label), 'OfficerType')
+ human_resource_costs = pd.concat([human_resource_costs, labelled_df])
+
+ human_resource_costs = prepare_cost_dataframe(human_resource_costs, _category_specific_group = 'OfficerType', _cost_category = 'human resources for health')
+
+ # Only preserve the draws and runs requested
+ if _draws is not None:
+ human_resource_costs = human_resource_costs[human_resource_costs.draw.isin(_draws)]
+ if _runs is not None:
+ human_resource_costs = human_resource_costs[human_resource_costs.run.isin(_runs)]
+
+ # %%
+ # 2. Consumables cost
+ #------------------------
+ print("Now estimating Consumables costs...")
+ def get_quantity_of_consumables_dispensed(results_folder):
+ def get_counts_of_items_requested(_df):
+ _df = drop_outside_period(_df)
+ counts_of_used = defaultdict(lambda: defaultdict(int))
+ counts_of_not_available = defaultdict(lambda: defaultdict(int))
+
+ for _, row in _df.iterrows():
+ date = row['date']
+ for item, num in row['Item_Used'].items():
+ counts_of_used[date][item] += num
+ for item, num in row['Item_NotAvailable'].items():
+ counts_of_not_available[date][item] += num
+ used_df = pd.DataFrame(counts_of_used).fillna(0).astype(int).stack().rename('Used')
+ not_available_df = pd.DataFrame(counts_of_not_available).fillna(0).astype(int).stack().rename('Not_Available')
+
+ # Combine the two dataframes into one series with MultiIndex (date, item, availability_status)
+ combined_df = pd.concat([used_df, not_available_df], axis=1).fillna(0).astype(int)
+
+ # Convert to a pd.Series, as expected by the custom_generate_series function
+ return combined_df.stack()
+
+ cons_req = extract_results(
+ results_folder,
+ module='tlo.methods.healthsystem.summary',
+ key='Consumables',
+ custom_generate_series=get_counts_of_items_requested,
+ do_scaling=True)
+
+ cons_dispensed = cons_req.xs("Used", level=2) # only keep actual dispensed amount, i.e. when available
+ return cons_dispensed
+ # TODO Extract year of dispensing drugs
+
+ consumables_dispensed = get_quantity_of_consumables_dispensed(results_folder)
+ consumables_dispensed = consumables_dispensed.reset_index().rename(columns = {'level_0': 'Item_Code', 'level_1': 'year'})
+ consumables_dispensed[idx['year']] = pd.to_datetime(consumables_dispensed[idx['year']]).dt.year # Extract only year from date
+ consumables_dispensed[idx['Item_Code']] = pd.to_numeric(consumables_dispensed[idx['Item_Code']])
+ # Make a list of columns in the DataFrame pertaining to quantity dispensed
+ quantity_columns = consumables_dispensed.columns.to_list()
+ quantity_columns = [tup for tup in quantity_columns if tup not in [('Item_Code', ''), ('year', '')]]
+
+ # 2.1 Cost of consumables dispensed
+ #---------------------------------------------------------------------------------------------------------------
+ # Multiply number of items needed by cost of consumable
+ #consumables_dispensed.columns = consumables_dispensed.columns.get_level_values(0).str() + "_" + consumables_dispensed.columns.get_level_values(1) # Flatten multi-level columns for pandas merge
+ unit_price_consumable.columns = pd.MultiIndex.from_arrays([unit_price_consumable.columns, [''] * len(unit_price_consumable.columns)])
+ cost_of_consumables_dispensed = consumables_dispensed.merge(unit_price_consumable, on = idx['Item_Code'], validate = 'm:1', how = 'left')
+ price_column = 'Final_price_per_chosen_unit (USD, 2023)'
+ cost_of_consumables_dispensed[quantity_columns] = cost_of_consumables_dispensed[quantity_columns].multiply(
+ cost_of_consumables_dispensed[price_column], axis=0)
+
+ # 2.2 Cost of consumables stocked (quantity needed for what is dispensed)
+ #---------------------------------------------------------------------------------------------------------------
+ # Stocked amount should be higher than dispensed because of i. excess capacity, ii. theft, iii. expiry
+ # While there are estimates in the literature of what % these might be, we agreed that it is better to rely upon
+ # an empirical estimate based on OpenLMIS data
+ # Estimate the stock to dispensed ratio from OpenLMIS data
+ lmis_consumable_usage = pd.read_csv(path_for_consumable_resourcefiles / "ResourceFile_Consumables_availability_and_usage.csv")
+ # TODO Generate a smaller version of this file
+ # Collapse individual facilities
+ lmis_consumable_usage_by_item_level_month = lmis_consumable_usage.groupby(['category', 'item_code', 'district', 'fac_type_tlo', 'month'])[['closing_bal', 'dispensed', 'received']].sum()
+ df = lmis_consumable_usage_by_item_level_month # Drop rows where monthly OpenLMIS data wasn't available
+ df = df.loc[df.index.get_level_values('month') != "Aggregate"]
+ # Opening balance in January is the closing balance for the month minus what was received during the month plus what was dispensed
+ opening_bal_january = df.loc[df.index.get_level_values('month') == 'January', 'closing_bal'] + \
+ df.loc[df.index.get_level_values('month') == 'January', 'dispensed'] - \
+ df.loc[df.index.get_level_values('month') == 'January', 'received']
+ closing_bal_december = df.loc[df.index.get_level_values('month') == 'December', 'closing_bal']
+ # the consumable inflow during the year is the opening balance in January + what was received throughout the year - what was transferred to the next year (i.e. closing bal of December)
+ total_consumables_inflow_during_the_year = df.loc[df.index.get_level_values('month') != 'January', 'received'].groupby(level=[0,1,2,3]).sum() +\
+ opening_bal_january.reset_index(level='month', drop=True) -\
+ closing_bal_december.reset_index(level='month', drop=True)
+ total_consumables_outflow_during_the_year = df['dispensed'].groupby(level=[0,1,2,3]).sum()
+ inflow_to_outflow_ratio = total_consumables_inflow_during_the_year.div(total_consumables_outflow_during_the_year, fill_value=1)
+
+ # Edit outlier ratios
+ inflow_to_outflow_ratio.loc[inflow_to_outflow_ratio < 1] = 1 # Ratio can't be less than 1
+ inflow_to_outflow_ratio.loc[inflow_to_outflow_ratio > inflow_to_outflow_ratio.quantile(0.95)] = inflow_to_outflow_ratio.quantile(0.95) # Trim values greater than the 95th percentile
+ average_inflow_to_outflow_ratio_ratio = inflow_to_outflow_ratio.mean() # Use average where item-specific ratio is not available
+
+ # Multiply number of items needed by cost of consumable
+ inflow_to_outflow_ratio_by_consumable = inflow_to_outflow_ratio.groupby(level='item_code').mean()
+ excess_stock_ratio = inflow_to_outflow_ratio_by_consumable - 1
+ excess_stock_ratio = excess_stock_ratio.reset_index().rename(columns = {0: 'excess_stock_proportion_of_dispensed'})
+ # TODO Consider whether a more disaggregated version of the ratio dictionary should be applied
+ cost_of_excess_consumables_stocked = consumables_dispensed.merge(unit_price_consumable, left_on = 'Item_Code', right_on = 'Item_Code', validate = 'm:1', how = 'left')
+ excess_stock_ratio.columns = pd.MultiIndex.from_arrays([excess_stock_ratio.columns, [''] * len(excess_stock_ratio.columns)]) # TODO convert this into a funciton
+ cost_of_excess_consumables_stocked = cost_of_excess_consumables_stocked.merge(excess_stock_ratio, left_on = 'Item_Code', right_on = 'item_code', validate = 'm:1', how = 'left')
+ cost_of_excess_consumables_stocked.loc[cost_of_excess_consumables_stocked.excess_stock_proportion_of_dispensed.isna(), 'excess_stock_proportion_of_dispensed'] = average_inflow_to_outflow_ratio_ratio - 1# TODO disaggregate the average by program
+ cost_of_excess_consumables_stocked[quantity_columns] = cost_of_excess_consumables_stocked[quantity_columns].multiply(cost_of_excess_consumables_stocked[idx[price_column]], axis=0)
+ cost_of_excess_consumables_stocked[quantity_columns] = cost_of_excess_consumables_stocked[quantity_columns].multiply(cost_of_excess_consumables_stocked[idx['excess_stock_proportion_of_dispensed']], axis=0)
+
+ # 2.3 Store all consumable costs in one standard format dataframe
+ #---------------------------------------------------------------------------------------------------------------
+ # Function to melt and label the cost category
+ consumables_dict = pd.read_csv(path_for_consumable_resourcefiles / 'ResourceFile_Consumables_Items_and_Packages.csv', low_memory=False,
+ encoding="ISO-8859-1")[['Items','Item_Code']]
+ consumables_dict = dict(zip(consumables_dict['Item_Code'], consumables_dict['Items']))
+ def melt_and_label_consumables_cost(_df, label):
+ multi_index = pd.MultiIndex.from_tuples(_df.columns)
+ _df.columns = multi_index
+ # Select 'Item_Code', 'year', and all columns where both levels of the MultiIndex are numeric (these are the (draw,run) columns with cost values)
+ selected_columns = [col for col in _df.columns if
+ (col[0] in ['Item_Code', 'year']) or (isinstance(col[0], int) and isinstance(col[1], int))]
+ _df = _df[selected_columns] # Subset the dataframe with the selected columns
+
+ # reshape dataframe and assign 'draw' and 'run' as the correct column headers
+ melted_df = pd.melt(_df, id_vars=['year', 'Item_Code']).rename(columns = {'variable_0': 'draw', 'variable_1': 'run'})
+ # Replace item_code with consumable_name_tlo
+ melted_df['consumable'] = melted_df['Item_Code'].map(consumables_dict)
+ melted_df['cost_subcategory'] = label
+ melted_df['Facility_Level'] = 'all' #TODO this is temporary until 'tlo.methods.healthsystem.summary' only logs consumable at the aggregate level
+ melted_df = melted_df.rename(columns = {'value': 'cost'})
+ return melted_df
+
+ def disaggregate_separately_managed_medical_supplies_from_consumable_costs(_df,
+ _consumables_dict, # This is a dictionary mapping codes to names
+ list_of_unique_medical_products):
+ reversed_consumables_dict = {value: key for key, value in _consumables_dict.items()} # reverse dictionary to map names to codes
+ new_df = _df.copy()
+ new_df['item_code'] = new_df['consumable'].map(reversed_consumables_dict)
+ cost_of_consumables = new_df[~new_df['item_code'].isin(list_of_unique_medical_products)]
+ cost_of_separately_managed_medical_supplies = new_df[new_df['item_code'].isin(list_of_unique_medical_products)]
+ cost_of_separately_managed_medical_supplies['cost_subcategory'] = cost_of_separately_managed_medical_supplies['cost_subcategory'].replace(
+ {'consumables_dispensed': 'separately_managed_medical_supplies_dispensed', 'consumables_stocked': 'separately_managed_medical_supplies_stocked'}, regex=True)
+ return cost_of_consumables.drop(columns = 'item_code'), cost_of_separately_managed_medical_supplies.drop(columns = 'item_code')
+
+ separately_managed_medical_supplies = [127, 141, 161] # Oxygen, Blood, IRS
+ cost_of_consumables_dispensed, cost_of_separately_managed_medical_supplies_dispensed = disaggregate_separately_managed_medical_supplies_from_consumable_costs(_df = retain_relevant_column_subset(melt_and_label_consumables_cost(cost_of_consumables_dispensed, 'cost_of_consumables_dispensed'), 'consumable'),
+ _consumables_dict = consumables_dict,
+ list_of_unique_medical_products = separately_managed_medical_supplies)
+ cost_of_excess_consumables_stocked, cost_of_separately_managed_medical_supplies_excess_stock = disaggregate_separately_managed_medical_supplies_from_consumable_costs(_df = retain_relevant_column_subset(melt_and_label_consumables_cost(cost_of_excess_consumables_stocked, 'cost_of_excess_consumables_stocked'), 'consumable'),
+ _consumables_dict=consumables_dict,
+ list_of_unique_medical_products=separately_managed_medical_supplies)
+
+ consumable_costs = pd.concat([cost_of_consumables_dispensed, cost_of_excess_consumables_stocked])
+
+ # 2.4 Supply chain costs
+ #---------------------------------------------------------------------------------------------------------------
+ # Assume that the cost of procurement, warehousing and distribution is a fixed proportion of consumable purchase costs
+ # The fixed proportion is based on Resource Mapping Expenditure data from 2018
+ resource_mapping_data = workbook_cost["resource_mapping_r7_summary"]
+ # Make sure values are numeric
+ expenditure_column = ['EXPENDITURE (USD) (Jul 2018 - Jun 2019)']
+ resource_mapping_data[expenditure_column] = resource_mapping_data[expenditure_column].apply(lambda x: pd.to_numeric(x, errors='coerce'))
+ supply_chain_expenditure = resource_mapping_data[resource_mapping_data['Cost Type'] == 'Supply Chain'][expenditure_column].sum()[0]
+ consumables_purchase_expenditure = resource_mapping_data[resource_mapping_data['Cost Type'] == 'Drugs and Commodities'][expenditure_column].sum()[
+ 0] + \
+ resource_mapping_data[resource_mapping_data['Cost Type'] == 'HIV Drugs and Commodities'][
+ expenditure_column].sum()[0]
+ supply_chain_cost_proportion = supply_chain_expenditure / consumables_purchase_expenditure
+
+ # Estimate supply chain costs based on the total consumable purchase cost calculated above
+ # Note that Oxygen, IRS, and Blood costs are already excluded because the unit_cost of these commodities already
+ # includes the procurement/production, storage and distribution costs
+ supply_chain_costs = (consumable_costs.groupby(['draw', 'run', 'year'])[
+ 'cost'].sum() * supply_chain_cost_proportion).reset_index()
+ # Assign relevant additional columns to match the format of the rest of consumables costs
+ supply_chain_costs['Facility_Level'] = 'all'
+ supply_chain_costs['consumable'] = 'supply chain (all consumables)'
+ supply_chain_costs['cost_subcategory'] = 'supply_chain'
+ assert set(supply_chain_costs.columns) == set(consumable_costs.columns)
+
+ # Append supply chain costs to the full consumable cost dataframe
+ consumable_costs = pd.concat([consumable_costs, supply_chain_costs])
+ other_costs = pd.concat([cost_of_separately_managed_medical_supplies_dispensed, cost_of_separately_managed_medical_supplies_excess_stock])
+
+ consumable_costs = prepare_cost_dataframe(consumable_costs, _category_specific_group = 'consumable', _cost_category = 'medical consumables')
+ other_costs = prepare_cost_dataframe(other_costs, _category_specific_group = 'consumable', _cost_category = 'medical consumables')
+
+ # Only preserve the draws and runs requested
+ if _draws is not None:
+ consumable_costs = consumable_costs[consumable_costs.draw.isin(_draws)]
+ other_costs = other_costs[other_costs.draw.isin(_draws)]
+ if _runs is not None:
+ consumable_costs = consumable_costs[consumable_costs.run.isin(_runs)]
+ other_costs = other_costs[other_costs.run.isin(_runs)]
+
+
+ # %%
+ # 3. Equipment cost
+ #--------------------------------------------
+ print("Now estimating Medical equipment costs...")
+ # Total cost of equipment required as per SEL (HSSP-III) only at facility IDs where it has been used in the simulation
+ # Get list of equipment used in the simulation by district and level
+ def get_equipment_used_by_district_and_facility(_df: pd.Series) -> pd.Series:
+ """Summarise the parsed logged-key results for one draw (as dataframe) into a pd.Series."""
+ _df = _df.pivot_table(index=['District', 'Facility_Level'],
+ values='EquipmentEverUsed',
+ aggfunc='first')
+ _df.index.name = 'year'
+ return _df['EquipmentEverUsed']
+
+ list_of_equipment_used_by_draw_and_run = extract_results(
+ Path(results_folder),
+ module='tlo.methods.healthsystem.summary',
+ key='EquipmentEverUsed_ByFacilityID',
+ custom_generate_series=get_equipment_used_by_district_and_facility,
+ do_scaling=False,
+ )
+ for col in list_of_equipment_used_by_draw_and_run.columns:
+ list_of_equipment_used_by_draw_and_run[col] = list_of_equipment_used_by_draw_and_run[col].apply(ast.literal_eval)
+
+ # Initialize an empty DataFrame
+ equipment_cost_across_sim = pd.DataFrame()
+
+ # Extract equipment cost for each draw and run
+ for d in _draws:
+ for r in _runs:
+ print(f"Processing draw {d} and run {r} of equipment costs")
+ # Extract a list of equipment which was used at each facility level within each district
+ equipment_used = {district: {level: [] for level in fac_levels} for district in list(district_dict.values())} # create a dictionary with a key for each district and facility level
+ list_of_equipment_used_by_current_draw_and_run = list_of_equipment_used_by_draw_and_run[(d, r)].reset_index()
+ for dist in list(district_dict.values()):
+ for level in fac_levels:
+ equipment_used_subset = list_of_equipment_used_by_current_draw_and_run[(list_of_equipment_used_by_current_draw_and_run['District'] == dist) & (list_of_equipment_used_by_current_draw_and_run['Facility_Level'] == level)]
+ equipment_used_subset.columns = ['District', 'Facility_Level', 'EquipmentEverUsed']
+ equipment_used[dist][level] = set().union(*equipment_used_subset['EquipmentEverUsed'])
+ equipment_used = pd.concat({
+ k: pd.DataFrame.from_dict(v, 'index') for k, v in equipment_used.items()},
+ axis=0)
+ full_list_of_equipment_used = set(equipment_used.values.flatten())
+ full_list_of_equipment_used = set(filter(pd.notnull, full_list_of_equipment_used))
+
+ equipment_df = pd.DataFrame()
+ equipment_df.index = equipment_used.index
+ for item in full_list_of_equipment_used:
+ equipment_df[str(item)] = 0
+ for dist_fac_index in equipment_df.index:
+ equipment_df.loc[equipment_df.index == dist_fac_index, str(item)] = equipment_used[equipment_used.index == dist_fac_index].isin([item]).any(axis=1)
+ #equipment_df.to_csv('./outputs/equipment_use.csv')
+
+ equipment_df = equipment_df.reset_index().rename(columns = {'level_0' : 'District', 'level_1': 'Facility_Level'})
+ equipment_df = pd.melt(equipment_df, id_vars = ['District', 'Facility_Level']).rename(columns = {'variable': 'Item_code', 'value': 'whether_item_was_used'})
+ equipment_df['Item_code'] = pd.to_numeric(equipment_df['Item_code'])
+ # Merge the count of facilities by district and level
+ equipment_df = equipment_df.merge(mfl[['District', 'Facility_Level','Facility_Count']], on = ['District', 'Facility_Level'], how = 'left')
+ equipment_df.loc[equipment_df.Facility_Count.isna(), 'Facility_Count'] = 0
+
+ # Because levels 1b and 2 are collapsed together, we assume that the same equipment is used by level 1b as that recorded for level 2
+ def update_itemuse_for_level1b_using_level2_data(_df):
+ # Create a list of District and Item_code combinations for which use == True
+ list_of_equipment_used_at_level2 = _df[(_df.Facility_Level == '2') & (_df['whether_item_was_used'] == True)][['District', 'Item_code']]
+ # Now update the 'whether_item_was_used' for 'Facility_Level' == '1b' to match that of level '2'
+ _df.loc[
+ (_df['Facility_Level'] == '1b') &
+ (_df[['District', 'Item_code']].apply(tuple, axis=1).isin(
+ list_of_equipment_used_at_level2.apply(tuple, axis=1))),
+ 'whether_item_was_used'
+ ] = True
+
+ return _df
+
+ equipment_df = update_itemuse_for_level1b_using_level2_data(equipment_df)
+
+ # Merge the two datasets to calculate cost
+ equipment_cost = pd.merge(equipment_df, unit_cost_equipment[['Item_code', 'Equipment_tlo', 'Facility_Level', 'Quantity', 'replacement_cost_annual', 'service_fee_annual', 'spare_parts_annual', 'major_corrective_maintenance_cost_annual']],
+ on = ['Item_code', 'Facility_Level'], how = 'left', validate = "m:1")
+ categories_of_equipment_cost = ['replacement_cost', 'service_fee', 'spare_parts', 'major_corrective_maintenance_cost']
+ for cost_category in categories_of_equipment_cost:
+ # Rename unit cost columns
+ unit_cost_column = cost_category + '_annual_unit'
+ equipment_cost = equipment_cost.rename(columns = {cost_category + '_annual':unit_cost_column })
+ equipment_cost[cost_category + '_annual_total'] = equipment_cost[cost_category + '_annual_unit'] * equipment_cost['whether_item_was_used'] * equipment_cost['Quantity'] * equipment_cost['Facility_Count']
+ equipment_cost['year'] = final_year_of_simulation - 1
+ if equipment_cost_across_sim.empty:
+ equipment_cost_across_sim = equipment_cost.groupby(['year', 'Facility_Level', 'Equipment_tlo'])[[item + '_annual_total' for item in categories_of_equipment_cost]].sum()
+ equipment_cost_across_sim['draw'] = d
+ equipment_cost_across_sim['run'] = r
+ else:
+ equipment_cost_for_current_sim = equipment_cost.groupby(['year', 'Facility_Level', 'Equipment_tlo'])[[item + '_annual_total' for item in categories_of_equipment_cost]].sum()
+ equipment_cost_for_current_sim['draw'] = d
+ equipment_cost_for_current_sim['run'] = r
+ # Concatenate the results
+ equipment_cost_across_sim = pd.concat([equipment_cost_across_sim, equipment_cost_for_current_sim], axis=0)
+
+ equipment_costs = pd.melt(equipment_cost_across_sim.reset_index(),
+ id_vars=['draw', 'run', 'Facility_Level', 'Equipment_tlo'], # Columns to keep
+ value_vars=[col for col in equipment_cost_across_sim.columns if col.endswith('_annual_total')], # Columns to unpivot
+ var_name='cost_subcategory', # New column name for the 'sub-category' of cost
+ value_name='cost') # New column name for the values
+
+ # Assume that the annual costs are constant each year of the simulation
+ equipment_costs = pd.concat([equipment_costs.assign(year=year) for year in years])
+ # TODO If the logger is updated to include year, we may wish to calculate equipment costs by year - currently we assume the same annuitised equipment cost each year
+ equipment_costs = equipment_costs.reset_index(drop=True)
+ equipment_costs = equipment_costs.rename(columns = {'Equipment_tlo': 'Equipment'})
+ equipment_costs = prepare_cost_dataframe(equipment_costs, _category_specific_group = 'Equipment', _cost_category = 'medical equipment')
+
+ # 4. Facility running costs
+ # Average running costs by facility level and district times the number of facilities in the simulation
+ # Convert unit_costs to long format
+ unit_cost_fac_operations = pd.melt(
+ unit_cost_fac_operations,
+ id_vars=["Facility_Level"], # Columns to keep as identifiers
+ var_name="operating_cost_type", # Name for the new 'cost_category' column
+ value_name="unit_cost" # Name for the new 'cost' column
+ )
+ unit_cost_fac_operations['Facility_Level'] = unit_cost_fac_operations['Facility_Level'].astype(str)
+ fac_count_by_level = mfl[['Facility_Level', 'Facility_Count']].groupby(['Facility_Level']).sum().reset_index()
+
+ facility_operation_cost = pd.merge(unit_cost_fac_operations, fac_count_by_level, on = 'Facility_Level', how = 'left', validate = 'm:m')
+ facility_operation_cost['Facility_Count'] = facility_operation_cost['Facility_Count'].fillna(0).astype(int)
+ facility_operation_cost['cost'] = facility_operation_cost['unit_cost'] * facility_operation_cost['Facility_Count']
+
+ # Duplicate the same set of facility operation costs for all draws and runs
+ # Create the Cartesian product of `_draws` and `_runs`
+ combinations = list(itertools.product(_draws, _runs))
+ comb_df = pd.DataFrame(combinations, columns=["draw", "run"])
+ facility_operation_cost = facility_operation_cost.merge(comb_df, how="cross")
+ facility_operation_cost['cost_category'] = 'Facility operating cost'
+ operating_cost_mapping = {'Electricity': 'utilities_and_maintenance', 'Water': 'utilities_and_maintenance', 'Cleaning':'utilities_and_maintenance',
+ 'Security':'utilities_and_maintenance', 'Building maintenance': 'building_maintenance',
+ 'Facility management': 'utilities_and_maintenance', 'Vehicle maintenance': 'vehicle_maintenance',
+ 'Ambulance fuel': 'fuel_for_ambulance', 'Food for inpatient cases': 'food_for_inpatient_care'}
+ facility_operation_cost['cost_subcategory'] = facility_operation_cost['operating_cost_type']
+ facility_operation_cost['cost_subcategory'] = facility_operation_cost['cost_subcategory'].map(operating_cost_mapping)
+ # Assume that the annual costs are constant each year of the simulation
+ facility_operation_cost = pd.concat([facility_operation_cost.assign(year=year) for year in years])
+
+ # Assume that the annual costs are constant each year of the simulation
+ facility_operation_cost = prepare_cost_dataframe(facility_operation_cost, _category_specific_group = 'operating_cost_type', _cost_category = 'facility operating cost')
+
+
+ # %%
+ # Store all costs in single dataframe
+ #--------------------------------------------
+ scenario_cost = pd.concat([human_resource_costs, consumable_costs, equipment_costs, other_costs, facility_operation_cost], ignore_index=True)
+ scenario_cost['cost'] = pd.to_numeric(scenario_cost['cost'], errors='coerce')
+
+ # Summarize costs
+ if summarize == True:
+ groupby_cols = [col for col in scenario_cost.columns if ((col != 'run') & (col != 'cost'))]
+ scenario_cost = pd.concat(
+ {
+ 'mean': scenario_cost.groupby(by=groupby_cols, sort=False)['cost'].mean(),
+ 'lower': scenario_cost.groupby(by=groupby_cols, sort=False)['cost'].quantile(0.025),
+ 'upper': scenario_cost.groupby(by=groupby_cols, sort=False)['cost'].quantile(0.975),
+ },
+ axis=1
+ )
+ scenario_cost = pd.melt(scenario_cost.reset_index(),
+ id_vars=groupby_cols, # Columns to keep
+ value_vars=['mean', 'lower', 'upper'], # Columns to unpivot
+ var_name='stat', # New column name for the 'sub-category' of cost
+ value_name='cost')
+
+ if _years is None:
+ return apply_discounting_to_cost_data(scenario_cost,_discount_rate)
+ else:
+ return apply_discounting_to_cost_data(scenario_cost[scenario_cost.year.isin(_years)],_discount_rate)
+
+# Define a function to summarize cost data from
+# Note that the dataframe needs to have draw as index and run as columns. if the dataframe is long with draw and run as index, then
+# first unstack the dataframe and subsequently apply the summarize function
+def summarize_cost_data(_df):
+ _df = _df.stack()
+ collapsed_df = _df.groupby(level='draw').agg([
+ 'mean',
+ ('lower', lambda x: x.quantile(0.025)),
+ ('upper', lambda x: x.quantile(0.975))
+ ])
+
+ collapsed_df = collapsed_df.unstack()
+ collapsed_df.index = collapsed_df.index.set_names('stat', level=0)
+ collapsed_df = collapsed_df.unstack(level='stat')
+ return collapsed_df
+
+# Estimate projected health spending
+####################################################
+def estimate_projected_health_spending(resourcefilepath: Path = None,
+ results_folder: Path = None,
+ _draws = None, _runs = None,
+ _years = None,
+ _discount_rate = 0,
+ _summarize = False):
+ # %% Gathering basic information
+ # Load basic simulation parameters
+ #-------------------------------------
+ log = load_pickled_dataframes(results_folder, 0, 0) # read from 1 draw and run
+ info = get_scenario_info(results_folder) # get basic information about the results
+ if _draws is None:
+ _draws = range(0, info['number_of_draws'])
+ if _runs is None:
+ _runs = range(0, info['runs_per_draw'])
+ final_year_of_simulation = max(log['tlo.methods.healthsystem.summary']['hsi_event_counts']['date']).year
+ first_year_of_simulation = min(log['tlo.methods.healthsystem.summary']['hsi_event_counts']['date']).year
+ if _years == None:
+ _years = list(range(first_year_of_simulation, final_year_of_simulation + 1))
+
+ # Load health spending per capita projections
+ #----------------------------------------
+ # Load health spending projections
+ workbook_cost = pd.read_excel((resourcefilepath / "costing/ResourceFile_Costing.xlsx"),
+ sheet_name=None)
+ health_spending_per_capita = workbook_cost["health_spending_projections"]
+ # Assign the fourth row as column names
+ health_spending_per_capita.columns = health_spending_per_capita.iloc[1]
+ health_spending_per_capita = health_spending_per_capita.iloc[2:].reset_index(drop=True)
+ health_spending_per_capita = health_spending_per_capita[
+ health_spending_per_capita.year.isin(list(range(2015, 2041)))]
+ total_health_spending_per_capita_mean = health_spending_per_capita[['year', 'total_mean']].set_index('year')
+
+ # Load population projections
+ # ----------------------------------------
+ def get_total_population(_df):
+ years_needed = [min(_years), max(_years)] # we only consider the population for the malaria scale-up period
+ # because those are the years relevant for malaria scale-up costing
+ _df['year'] = pd.to_datetime(_df['date']).dt.year
+ _df = _df[['year', 'total']]
+ assert set(_df.year.unique()).issuperset(years_needed), "Some years are not recorded."
+ return pd.Series(_df.loc[_df.year.between(*years_needed)].set_index('year')['total'])
+
+ total_population_by_year = extract_results(
+ results_folder,
+ module='tlo.methods.demography',
+ key='population',
+ custom_generate_series=get_total_population,
+ do_scaling=True
+ )
+ population_columns = total_population_by_year.columns
+
+ # Estimate total health spending
+ projected_health_spending = pd.merge(total_health_spending_per_capita_mean,
+ total_population_by_year,
+ left_index=True, right_index=True,how='inner')
+ projected_health_spending = projected_health_spending.apply(pd.to_numeric, errors='coerce')
+ projected_health_spending[population_columns] = projected_health_spending[population_columns].multiply(
+ projected_health_spending['total_mean'], axis=0)
+ projected_health_spending = projected_health_spending[population_columns]
+
+ # Apply discount rate
+ # Initial year and discount rate
+ initial_year = min(projected_health_spending.index.get_level_values('year').unique())
+ # Discount factor calculation
+ discount_factors = (1 + _discount_rate) ** (projected_health_spending.index.get_level_values('year') - initial_year)
+ # Apply the discount to the specified columns
+ projected_health_spending.loc[:, population_columns] = (
+ projected_health_spending[population_columns].div(discount_factors, axis=0))
+ # add across years
+ projected_health_spending = projected_health_spending.sum(axis = 0)
+ projected_health_spending.index = pd.MultiIndex.from_tuples(projected_health_spending.index, names=["draw", "run"])
+
+ if _summarize == True:
+ # Calculate the mean and 95% confidence intervals for each group
+ projected_health_spending = projected_health_spending.groupby(level="draw").agg(
+ mean=np.mean,
+ lower=lambda x: np.percentile(x, 2.5),
+ upper=lambda x: np.percentile(x, 97.5)
+ )
+ # Flatten the resulting DataFrame into a single-level MultiIndex Series
+ projected_health_spending = projected_health_spending.stack().rename_axis(["draw", "stat"]).rename("value")
+
+ return projected_health_spending.unstack()
+
+# Plot costs
+####################################################
+# 1. Stacked bar plot (Total cost + Cost categories)
+#----------------------------------------------------
+def do_stacked_bar_plot_of_cost_by_category(_df, _cost_category = 'all',
+ _disaggregate_by_subgroup: bool = False,
+ _year = 'all', _draws = None,
+ _scenario_dict: dict = None,
+ _outputfilepath: Path = None,
+ _add_figname_suffix = ''):
+ # Subset and Pivot the data to have 'Cost Sub-category' as columns
+ # Make a copy of the dataframe to avoid modifying the original
+ _df_mean = _df[_df.stat == 'mean'].copy()
+ _df_lower = _df[_df.stat == 'lower'].copy()
+ _df_upper = _df[_df.stat == 'upper'].copy()
+
+ # Subset the dataframes to keep the s=relevant categories for the plot
+ dfs = {"_df_mean": _df_mean, "_df_lower": _df_lower, "_df_upper": _df_upper} # create a dict of dataframes
+ for name, df in dfs.items():
+ dfs[name] = df.copy() # Choose the dataframe to modify
+ # Convert 'cost' to millions
+ dfs[name]['cost'] = dfs[name]['cost'] / 1e6
+ # Subset data
+ if _draws is not None:
+ dfs[name] = dfs[name][dfs[name].draw.isin(_draws)]
+ if _year != 'all':
+ dfs[name] = dfs[name][dfs[name]['year'].isin(_year)]
+ if _cost_category != 'all':
+ dfs[name] = dfs[name][dfs[name]['cost_category'] == _cost_category]
+
+ # Extract the updated DataFrames back from the dictionary
+ _df_mean, _df_lower, _df_upper = dfs["_df_mean"], dfs["_df_lower"], dfs["_df_upper"]
+
+ if _cost_category == 'all':
+ if (_disaggregate_by_subgroup == True):
+ raise ValueError(f"Invalid input for _disaggregate_by_subgroup: '{_disaggregate_by_subgroup}'. "
+ f"Value can be True only when plotting a specific _cost_category")
+ else:
+ pivot_mean = _df_mean.pivot_table(index='draw', columns='cost_category', values='cost', aggfunc='sum')
+ pivot_lower = _df_lower.pivot_table(index='draw', columns='cost_category', values='cost', aggfunc='sum')
+ pivot_upper = _df_upper.pivot_table(index='draw', columns='cost_category', values='cost', aggfunc='sum')
+ else:
+ if (_disaggregate_by_subgroup == True):
+ for name, df in dfs.items():
+ dfs[name] = df.copy() # Choose the dataframe to modify
+ # If sub-groups are more than 10 in number, then disaggregate the top 10 and group the rest into an 'other' category
+ if (len(dfs[name]['cost_subgroup'].unique()) > 10):
+ # Calculate total cost per subgroup
+ subgroup_totals = dfs[name].groupby('cost_subgroup')['cost'].sum()
+ # Identify the top 10 subgroups by cost
+ top_10_subgroups = subgroup_totals.nlargest(10).index.tolist()
+ # Label the remaining subgroups as 'other'
+ dfs[name]['cost_subgroup'] = dfs[name]['cost_subgroup'].apply(
+ lambda x: x if x in top_10_subgroups else 'All other items'
+ )
+
+ # Extract the updated DataFrames back from the dictionary
+ _df_mean, _df_lower, _df_upper = dfs["_df_mean"], dfs["_df_lower"], dfs["_df_upper"]
+
+ pivot_mean = _df_mean.pivot_table(index='draw', columns='cost_subgroup',
+ values='cost', aggfunc='sum')
+ pivot_lower = _df_lower.pivot_table(index='draw', columns='cost_subgroup',
+ values='cost', aggfunc='sum')
+ pivot_upper = _df_upper.pivot_table(index='draw', columns='cost_subgroup',
+ values='cost', aggfunc='sum')
+
+ plt_name_suffix = '_by_subgroup'
+ else:
+ pivot_mean = _df_mean.pivot_table(index='draw', columns='cost_subcategory', values='cost', aggfunc='sum')
+ pivot_lower = _df_lower.pivot_table(index='draw', columns='cost_subcategory', values='cost', aggfunc='sum')
+ pivot_upper = _df_upper.pivot_table(index='draw', columns='cost_subcategory', values='cost', aggfunc='sum')
+ plt_name_suffix = ''
+
+ # Sort pivot_df columns in ascending order by total cost
+ sorted_columns = pivot_mean.sum(axis=0).sort_values().index
+ pivot_mean = pivot_mean[sorted_columns]
+ pivot_lower = pivot_lower[sorted_columns]
+ pivot_upper = pivot_upper[sorted_columns]
+
+ # Error bars
+ lower_bounds = pivot_mean.sum(axis=1) - pivot_lower.sum(axis=1)
+ lower_bounds[lower_bounds<0] = 0
+ upper_bounds = pivot_upper.sum(axis=1) - pivot_mean.sum(axis=1)
+
+ if _cost_category == 'all':
+ # Predefined color mapping for cost categories
+ color_mapping = {
+ 'human resources for health': '#1f77b4', # Muted blue
+ 'medical consumables': '#ff7f0e', # Muted orange
+ 'medical equipment': '#2ca02c', # Muted green
+ 'other': '#d62728', # Muted red
+ 'facility operating cost': '#9467bd', # Muted purple
+ }
+ # Default color for unexpected categories
+ default_color = 'gray'
+ plt_name_suffix = ''
+
+ # Define custom colors for the bars
+ if _cost_category == 'all':
+ column_colors = [color_mapping.get(col, default_color) for col in sorted_columns]
+ # Plot the stacked bar chart with set colours
+ ax = pivot_mean.plot(kind='bar', stacked=True, figsize=(10, 6), color=column_colors)
+
+ # Add data labels
+ for c in ax.containers:
+ # Add label only if the value of the segment is > 1.20th of the ylim
+ max_y = ax.get_ylim()[1]
+ labels = [round(v.get_height(),1) if v.get_height() > max_y/20 else '' for v in c]
+ # remove the labels parameter if it's not needed for customized labels
+ ax.bar_label(c, labels=labels, label_type='center', fontsize='small')
+
+ # Add error bars
+ x_pos = np.arange(len(pivot_mean.index))
+ total_means = pivot_mean.sum(axis=1)
+ error_bars = [lower_bounds, upper_bounds]
+ ax.errorbar(x_pos, total_means, yerr=error_bars, fmt='o', color='black', capsize=5)
+
+ else:
+ # Plot the stacked bar chart without set colours
+ ax = pivot_mean.plot(kind='bar', stacked=True, figsize=(10, 6))
+
+ # Add data labels
+ for c in ax.containers:
+ # Add label only if the value of the segment is > 1.20th of the ylim
+ max_y = ax.get_ylim()[1]
+ labels = [round(v.get_height(),1) if v.get_height() > max_y/20 else '' for v in c]
+ # remove the labels parameter if it's not needed for customized labels
+ ax.bar_label(c, labels=labels, label_type='center', fontsize='small')
+
+ # Add error bars
+ x_pos = np.arange(len(pivot_mean.index))
+ total_means = pivot_mean.sum(axis=1)
+ error_bars = [lower_bounds, upper_bounds]
+ ax.errorbar(x_pos, total_means, yerr=error_bars, fmt='o', color='black', capsize=5)
+
+ # Set custom x-tick labels if _scenario_dict is provided
+ if _scenario_dict:
+ labels = [_scenario_dict.get(label, label) for label in pivot_mean.index]
+ else:
+ labels = pivot_mean.index.astype(str)
+
+ # Wrap x-tick labels for readability
+ wrapped_labels = [textwrap.fill(str(label), 20) for label in labels]
+ ax.set_xticklabels(wrapped_labels, rotation=45, ha='right', fontsize='small')
+
+ # Period included for plot title and name
+ if _year == 'all':
+ period = (f"{min(_df_mean['year'].unique())} - {max(_df_mean['year'].unique())}")
+ elif (len(_year) == 1):
+ period = (f"{_year[0]}")
+ else:
+ period = (f"{min(_year)} - {max(_year)}")
+
+ # Save plot
+ plt.xlabel('Scenario')
+ plt.ylabel('Cost (2023 USD), millions')
+
+ # Arrange the legend in the same ascending order
+ handles, labels = plt.gca().get_legend_handles_labels()
+ plt.legend(handles[::-1], labels[::-1], bbox_to_anchor=(1.05, 0.7), loc='center left', fontsize='small')
+
+ # Extend the y-axis by 25%
+ max_y = ax.get_ylim()[1]
+ ax.set_ylim(0, max_y*1.25)
+
+ # Save the plot with tight layout
+ plt.tight_layout(pad=2.0) # Ensure there is enough space for the legend
+ plt.subplots_adjust(right=0.8) # Adjust to ensure legend doesn't overlap
+
+ # Add gridlines and border
+ plt.grid(visible=True, which='major', linestyle='--', linewidth=0.5, color='gray')
+ #plt.rcParams['figure.facecolor'] = 'white'
+ plt.rcParams['figure.edgecolor'] = 'gray'
+ plt.rcParams['figure.frameon'] = True
+
+ plt.title(f'Costs by Scenario \n (Cost Category = {_cost_category} ; Period = {period})')
+ plt.savefig(_outputfilepath / f'stacked_bar_chart_{_cost_category}_{period}{plt_name_suffix}{_add_figname_suffix}.png', dpi=100,
+ bbox_inches='tight')
+ plt.close()
+
+# 2. Line plots of total costs
+#----------------------------------------------------
+# TODO: Check why line plot get save without a file name
+def do_line_plot_of_cost(_df, _cost_category='all',
+ _year='all', _draws=None,
+ disaggregate_by=None,
+ _outputfilepath: Path = None):
+ # Validate disaggregation options
+ valid_disaggregations = ['cost_category', 'cost_subcategory', 'cost_subgroup']
+ if disaggregate_by not in valid_disaggregations and disaggregate_by is not None:
+ raise ValueError(f"Invalid disaggregation option: {disaggregate_by}. Choose from {valid_disaggregations}.")
+
+ #
+ if ((_draws is None) or (len(_draws) > 1)) & (disaggregate_by is not None):
+ raise ValueError(f"The disaggregate_by option only works if only one draw is plotted, for exmaple _draws = [0]")
+
+ # Filter the dataframe by draws, if specified
+ subset_df = _df if _draws is None else _df[_df.draw.isin(_draws)]
+
+ # Filter by year if specified
+ if _year != 'all':
+ subset_df = subset_df[subset_df['year'].isin(_year)]
+
+ # Handle scenarios based on `_cost_category` and `disaggregate_by` conditions
+ if _cost_category == 'all':
+ if disaggregate_by == 'cost_subgroup':
+ raise ValueError("Cannot disaggregate by 'cost_subgroup' when `_cost_category='all'` due to data size.")
+ else:
+ # Filter subset_df by specific cost category if specified
+ subset_df = subset_df[subset_df['cost_category'] == _cost_category]
+
+ # Set grouping columns based on the disaggregation level
+ if disaggregate_by == 'cost_category':
+ groupby_columns = ['year', 'cost_category']
+ elif disaggregate_by == 'cost_subcategory':
+ groupby_columns = ['year', 'cost_subcategory']
+ elif disaggregate_by == 'cost_subgroup':
+ # If disaggregating by 'cost_subgroup' and there are more than 10 subgroups, limit to the top 10 + "Other"
+ if len(subset_df['cost_subgroup'].unique()) > 10:
+ # Calculate total cost per subgroup
+ subgroup_totals = subset_df[subset_df.stat == 'mean'].groupby('cost_subgroup')['cost'].sum()
+ # Identify the top 10 subgroups by cost
+ top_10_subgroups = subgroup_totals.nlargest(10).index.tolist()
+ # Reassign smaller subgroups to an "Other" category
+ subset_df['cost_subgroup'] = subset_df['cost_subgroup'].apply(
+ lambda x: x if x in top_10_subgroups else 'Other'
+ )
+ groupby_columns = ['year', 'cost_subgroup']
+ else:
+ groupby_columns = ['year']
+
+ # Extract mean, lower, and upper values for the plot
+ mean_values = subset_df[subset_df.stat == 'mean'].groupby(groupby_columns)['cost'].sum() / 1e6
+ lower_values = subset_df[subset_df.stat == 'lower'].groupby(groupby_columns)['cost'].sum() / 1e6
+ upper_values = subset_df[subset_df.stat == 'upper'].groupby(groupby_columns)['cost'].sum() / 1e6
+
+ # Prepare to store lines and labels for the legend
+ lines = []
+ labels = []
+
+ # Define a list of colors
+ if disaggregate_by == 'cost_category':
+ color_mapping = {
+ 'human resources for health': '#1f77b4', # Muted blue
+ 'medical consumables': '#ff7f0e', # Muted orange
+ 'medical equipment': '#2ca02c', # Muted green
+ 'other': '#d62728', # Muted red
+ 'facility operating cost': '#9467bd', # Muted purple
+ }
+ # Default color for unexpected categories
+ default_color = 'gray'
+ else:
+ # Define a list of colors to rotate through
+ colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'orange', 'purple', 'brown', 'gray'] # Add more colors as needed
+ color_cycle = iter(colors) # Create an iterator from the color list
+
+ # Plot each line for the disaggregated values
+ if disaggregate_by:
+ for disaggregate_value in mean_values.index.get_level_values(disaggregate_by).unique():
+ # Get mean, lower, and upper values for each disaggregated group
+ value_mean = mean_values.xs(disaggregate_value, level=disaggregate_by)
+ value_lower = lower_values.xs(disaggregate_value, level=disaggregate_by)
+ value_upper = upper_values.xs(disaggregate_value, level=disaggregate_by)
+
+ if disaggregate_by == 'cost_category':
+ color = color_mapping.get(disaggregate_value, default_color)
+ else:
+ # Get the next color from the cycle
+ color = next(color_cycle)
+
+ # Plot line for mean and shaded region for 95% CI
+ line, = plt.plot(value_mean.index, value_mean, marker='o', linestyle='-', color=color, label=f'{disaggregate_value} - Mean')
+ plt.fill_between(value_mean.index, value_lower, value_upper, color=color, alpha=0.2)
+
+ # Append to lines and labels for sorting later
+ lines.append(line)
+ labels.append(disaggregate_value)
+ else:
+ line, = plt.plot(mean_values.index, mean_values, marker='o', linestyle='-', color='b', label='Mean')
+ plt.fill_between(mean_values.index, lower_values, upper_values, color='b', alpha=0.2)
+
+ # Append to lines and labels for sorting later
+ lines.append(line)
+ labels.append('Mean')
+
+ # Sort the legend based on total costs
+ total_costs = {label: mean_values.xs(label, level=disaggregate_by).sum() for label in labels}
+ sorted_labels = sorted(total_costs.keys(), key=lambda x: total_costs[x])
+
+ # Reorder lines based on sorted labels
+ handles = [lines[labels.index(label)] for label in sorted_labels]
+
+ # Define period for plot title
+ if _year == 'all':
+ period = f"{min(subset_df['year'].unique())} - {max(subset_df['year'].unique())}"
+ elif len(_year) == 1:
+ period = str(_year[0])
+ else:
+ period = f"{min(_year)} - {max(_year)}"
+
+ # Set labels, legend, and title
+ # Add gridlines and border
+ plt.grid(visible=True, which='major', linestyle='--', linewidth=0.5, color='gray')
+ plt.xlabel('Year')
+ plt.ylabel('Cost (2023 USD), millions')
+ plt.legend(handles[::-1], sorted_labels[::-1], bbox_to_anchor=(1.05, 1), loc='upper left')
+ plot_title = f'Total input cost \n (Category = {_cost_category}, Period = {period})'
+ plt.title(plot_title)
+
+ # Save plot with a proper filename
+ if disaggregate_by is None:
+ filename_suffix = ""
+ else:
+ filename_suffix = f"_by_{disaggregate_by}"
+
+ draw_suffix = 'all' if _draws is None else str(_draws)
+ filename = f'trend_{_cost_category}_{period}{filename_suffix}_draw-{draw_suffix}.png'
+ plt.savefig(_outputfilepath / filename, dpi=100, bbox_inches='tight')
+ plt.close()
+
+# Treemap by category subgroup
+#-----------------------------------------------------------------------------------------------
+def create_summary_treemap_by_cost_subgroup(_df, _cost_category = None, _draw = None, _year = 'all',
+ _color_map = None, _label_fontsize = 10,
+ _outputfilepath: Path = None):
+ # Function to wrap text to fit within treemap rectangles
+ def wrap_text(text, width=15):
+ return "\n".join(textwrap.wrap(text, width))
+
+ valid_cost_categories = ['human resources for health', 'medical consumables',
+ 'medical equipment', 'facility operating cost']
+ if _cost_category == None:
+ raise ValueError(f"Specify one of the following as _cost_category - {valid_cost_categories})")
+ elif _cost_category not in valid_cost_categories:
+ raise ValueError(f"Invalid input for _cost_category: '{_cost_category}'. "
+ f"Specify one of the following - {valid_cost_categories})")
+ else:
+ _df = _df[_df['cost_category'] == _cost_category]
+
+ if _draw != None:
+ _df = _df[_df.draw == _draw]
+
+ # Remove non-specific subgroup for consumables
+ if _cost_category == 'medical consumables':
+ _df = _df[~(_df.cost_subgroup == 'supply chain (all consumables)')]
+
+ # Create summary dataframe for treemap
+ _df = _df.groupby('cost_subgroup')['cost'].sum().reset_index()
+ _df = _df.sort_values(by="cost", ascending=False)
+ top_10 = _df.iloc[:10]
+
+ if (len(_df['cost_subgroup'].unique()) > 10):
+ # Step 2: Group all other consumables into "Other"
+ other_cost = _df.iloc[10:]["cost"].sum()
+ top_10 = top_10.append({"cost_subgroup": "Other", "cost": other_cost}, ignore_index=True)
+
+ # Prepare data for the treemap
+ total_cost = top_10["cost"].sum()
+ top_10["proportion"] = top_10["cost"]/total_cost
+ sizes = top_10["cost"]
+
+ # Handle color map
+ if _color_map is None:
+ # Generate automatic colors if no color map is provided
+ auto_colors = plt.cm.Paired.colors
+ color_cycle = cycle(auto_colors) # Cycle through the automatic colors
+ color_map = {subgroup: next(color_cycle) for subgroup in top_10["cost_subgroup"]}
+ else:
+ # Use the provided color map, fallback to a default color for missing subgroups
+ fallback_color = '#cccccc'
+ color_map = {subgroup: _color_map.get(subgroup, fallback_color) for subgroup in top_10["cost_subgroup"]}
+
+ # Get colors for each subgroup
+ colors = [color_map[subgroup] for subgroup in top_10["cost_subgroup"]]
+
+ # Exclude labels for small proportions
+ labels = [
+ f"{wrap_text(name)}\n${round(cost, 1)}m\n({round(prop * 100, 1)}%)"
+ if prop >= 0.01 else ""
+ for name, cost, prop in zip(top_10["cost_subgroup"], top_10["cost"] / 1e6, top_10["proportion"])
+ ]
+ # Period included for plot title and name
+ if _year == 'all':
+ period = (f"{min(_df['year'].unique())} - {max(_df['year'].unique())}")
+ elif (len(_year) == 1):
+ period = (f"{_year[0]}")
+ else:
+ period = (f"{min(_year)} - {max(_year)}")
+
+ # Plot the treemap
+ plt.figure(figsize=(12, 8))
+ squarify.plot(sizes=sizes, label=labels, alpha=0.8, color=colors, text_kwargs={'fontsize': _label_fontsize})
+ plt.axis("off")
+ plt.title(f'{_cost_category} ; Period = {period}')
+ plt.savefig(_outputfilepath / f'treemap_{_cost_category}_[{_draw}]_{period}.png',
+ dpi=100,
+ bbox_inches='tight')
+ plt.close()
+
+# Plot ROI
+# TODO update this function to include an input for the monetary value of DALY
+def generate_roi_plots(_monetary_value_of_incremental_health: pd.DataFrame,
+ _incremental_input_cost: pd.DataFrame,
+ _scenario_dict: dict,
+ _outputfilepath: Path,
+ _value_of_life_suffix = ''):
+ # Calculate maximum ability to pay for implementation
+ max_ability_to_pay_for_implementation = (_monetary_value_of_incremental_health - _incremental_input_cost).clip(
+ lower=0.0) # monetary value - change in costs
+
+ # Iterate over each draw in monetary_value_of_incremental_health
+ for draw_index, row in _monetary_value_of_incremental_health.iterrows():
+ print("Plotting ROI for draw ", draw_index)
+ # Initialize an empty DataFrame to store values for each 'run'
+ all_run_values = pd.DataFrame()
+
+ # Create an array of implementation costs ranging from 0 to the max value of max ability to pay for the current draw
+ implementation_costs = np.linspace(0, max_ability_to_pay_for_implementation.loc[draw_index].max(), 50)
+
+ # Retrieve the corresponding row from incremental_scenario_cost for the same draw
+ incremental_scenario_cost_row = _incremental_input_cost.loc[draw_index]
+
+ # Calculate the values for each individual run
+ for run in incremental_scenario_cost_row.index: # Assuming 'run' columns are labeled by numbers
+ # Calculate the total costs for the current run
+ total_costs = implementation_costs + incremental_scenario_cost_row[run]
+
+ # Initialize run_values as an empty series with the same index as total_costs
+ run_values = pd.Series(index=total_costs, dtype=float)
+
+ # For negative total_costs, set corresponding run_values to infinity
+ run_values[total_costs < 0] = np.inf
+
+ # For non-negative total_costs, calculate the metric and clip at 0
+ non_negative_mask = total_costs >= 0
+ run_values[non_negative_mask] = np.clip(
+ (row[run] - total_costs[non_negative_mask]) / total_costs[non_negative_mask],
+ 0,
+ None
+ )
+
+ # Create a DataFrame with index as (draw_index, run) and columns as implementation costs
+ run_values = run_values.values # remove index and convert to array
+ run_df = pd.DataFrame([run_values], index=pd.MultiIndex.from_tuples([(draw_index, run)], names=['draw', 'run']),
+ columns=implementation_costs)
+
+ # Append the run DataFrame to all_run_values
+ all_run_values = pd.concat([all_run_values, run_df])
+
+ # Replace inf with NaN temporarily to handle quantile calculation correctly
+ temp_data = all_run_values.replace([np.inf, -np.inf], np.nan)
+
+ collapsed_data = temp_data.groupby(level='draw').agg([
+ 'mean',
+ ('lower', lambda x: x.quantile(0.025)),
+ ('upper', lambda x: x.quantile(0.975))
+ ])
+
+ # Revert the NaNs back to inf
+ collapsed_data = collapsed_data.replace([np.nan], np.inf)
+
+ collapsed_data = collapsed_data.unstack()
+ collapsed_data.index = collapsed_data.index.set_names('implementation_cost', level=0)
+ collapsed_data.index = collapsed_data.index.set_names('stat', level=1)
+ collapsed_data = collapsed_data.reset_index().rename(columns = {0: 'roi'})
+ #collapsed_data = collapsed_data.reorder_levels(['draw', 'stat', 'implementation_cost'])
+
+ # Divide rows by the sum of implementation costs and incremental input cost
+ mean_values = collapsed_data[collapsed_data['stat'] == 'mean'][['implementation_cost', 'roi']]
+ lower_values = collapsed_data[collapsed_data['stat'] == 'lower'][['implementation_cost', 'roi']]
+ upper_values = collapsed_data[collapsed_data['stat'] == 'upper'][['implementation_cost', 'roi']]
+
+ fig, ax = plt.subplots() # Create a figure and axis
+
+ # Plot mean line
+ plt.plot(implementation_costs / 1e6, mean_values['roi'], label=f'{_scenario_dict[draw_index]}')
+ # Plot the confidence interval as a shaded region
+ plt.fill_between(implementation_costs / 1e6, lower_values['roi'], upper_values['roi'], alpha=0.2)
+
+ # Set y-axis limit to upper max + 500
+ ax.set_ylim(0, mean_values[~np.isinf(mean_values.roi)]['roi'].max()*(1+0.05))
+
+ plt.xlabel('Implementation cost, millions')
+ plt.ylabel('Return on Investment')
+ plt.title('Return on Investment of scenario at different levels of implementation cost')
+
+ monetary_value_of_incremental_health_summarized = summarize_cost_data(_monetary_value_of_incremental_health)
+ incremental_scenario_cost_row_summarized = incremental_scenario_cost_row.agg(
+ mean='mean',
+ lower=lambda x: x.quantile(0.025),
+ upper=lambda x: x.quantile(0.975))
+
+ plt.text(x=0.95, y=0.8,
+ s=f"Monetary value of incremental health = \n USD {round(monetary_value_of_incremental_health_summarized.loc[draw_index]['mean'] / 1e6, 2)}m (USD {round(monetary_value_of_incremental_health_summarized.loc[draw_index]['lower'] / 1e6, 2)}m-{round(monetary_value_of_incremental_health_summarized.loc[draw_index]['upper'] / 1e6, 2)}m);\n "
+ f"Incremental input cost of scenario = \n USD {round(incremental_scenario_cost_row_summarized['mean'] / 1e6, 2)}m (USD {round(incremental_scenario_cost_row_summarized['lower'] / 1e6, 2)}m-{round(incremental_scenario_cost_row_summarized['upper'] / 1e6, 2)}m)",
+ horizontalalignment='right', verticalalignment='top', transform=plt.gca().transAxes, fontsize=9,
+ weight='bold', color='black')
+
+ # Show legend
+ plt.legend()
+ # Save
+ plt.savefig(_outputfilepath / f'draw{draw_index}_{_scenario_dict[draw_index]}_ROI_at_{_value_of_life_suffix}.png', dpi=100,
+ bbox_inches='tight')
+ plt.close()
+
+def generate_multiple_scenarios_roi_plot(_monetary_value_of_incremental_health: pd.DataFrame,
+ _incremental_input_cost: pd.DataFrame,
+ _draws:None,
+ _scenario_dict: dict,
+ _outputfilepath: Path,
+ _value_of_life_suffix = '',
+ _y_axis_lim = None,
+ _plot_vertical_lines_at: list = None,
+ _year_suffix = '',
+ _projected_health_spending = None,
+ _draw_colors = None):
+ # Default color mapping if not provided
+ if _draw_colors is None:
+ _draw_colors = {draw: color for draw, color in zip(_draws, plt.cm.tab10.colors[:len(_draws)])}
+
+ # Calculate maximum ability to pay for implementation
+ _monetary_value_of_incremental_health = _monetary_value_of_incremental_health[_monetary_value_of_incremental_health.index.get_level_values('draw').isin(_draws)]
+ _incremental_input_cost = _incremental_input_cost[_incremental_input_cost.index.get_level_values('draw').isin(_draws)]
+ max_ability_to_pay_for_implementation = (_monetary_value_of_incremental_health - _incremental_input_cost).clip(lower=0.0) # monetary value - change in costs
+
+ # Create a figure and axis to plot all draws together
+ fig, ax = plt.subplots(figsize=(10, 6))
+
+ # Store ROI values for specific costs
+ max_roi = []
+ roi_at_costs = {cost: [] for cost in (_plot_vertical_lines_at or [])}
+
+ # Iterate over each draw in monetary_value_of_incremental_health
+ for draw_index, row in _monetary_value_of_incremental_health.iterrows():
+ print("Plotting ROI for draw ", draw_index)
+ # Initialize an empty DataFrame to store values for each 'run'
+ all_run_values = pd.DataFrame()
+
+ # Create an array of implementation costs ranging from 0 to the max value of max ability to pay for the current draw
+ implementation_costs = np.linspace(0, max_ability_to_pay_for_implementation.loc[draw_index].max(), 50)
+ # Add fixed values for ROI ratio calculation
+ additional_costs = np.array([1_000_000_000, 3_000_000_000])
+ implementation_costs = np.sort(np.unique(np.concatenate([implementation_costs, additional_costs])))
+
+ # Retrieve the corresponding row from incremental_scenario_cost for the same draw
+ incremental_scenario_cost_row = _incremental_input_cost.loc[draw_index]
+
+ # Calculate the values for each individual run
+ for run in incremental_scenario_cost_row.index: # Assuming 'run' columns are labeled by numbers
+ # Calculate the total costs for the current run
+ total_costs = implementation_costs + incremental_scenario_cost_row[run]
+
+ # Initialize run_values as an empty series with the same index as total_costs
+ run_values = pd.Series(index=total_costs, dtype=float)
+
+ # For negative total_costs, set corresponding run_values to infinity
+ run_values[total_costs < 0] = np.inf
+
+ # For non-negative total_costs, calculate the metric and clip at 0
+ non_negative_mask = total_costs >= 0
+ run_values[non_negative_mask] = np.clip(
+ (row[run] - total_costs[non_negative_mask]) / total_costs[non_negative_mask],
+ 0,
+ None
+ )
+
+ # Create a DataFrame with index as (draw_index, run) and columns as implementation costs
+ run_values = run_values.values # remove index and convert to array
+ run_df = pd.DataFrame([run_values], index=pd.MultiIndex.from_tuples([(draw_index, run)], names=['draw', 'run']),
+ columns=implementation_costs)
+
+ # Append the run DataFrame to all_run_values
+ all_run_values = pd.concat([all_run_values, run_df])
+
+ # Replace inf with NaN temporarily to handle quantile calculation correctly
+ temp_data = all_run_values.replace([np.inf, -np.inf], np.nan)
+
+ collapsed_data = temp_data.groupby(level='draw').agg([
+ 'mean',
+ ('lower', lambda x: x.quantile(0.025)),
+ ('upper', lambda x: x.quantile(0.975))
+ ])
+
+ # Revert the NaNs back to inf
+ collapsed_data = collapsed_data.replace([np.nan], np.inf)
+
+ collapsed_data = collapsed_data.unstack()
+ collapsed_data.index = collapsed_data.index.set_names('implementation_cost', level=0)
+ collapsed_data.index = collapsed_data.index.set_names('stat', level=1)
+ collapsed_data = collapsed_data.reset_index().rename(columns = {0: 'roi'})
+
+ # Divide rows by the sum of implementation costs and incremental input cost
+ mean_values = collapsed_data[collapsed_data['stat'] == 'mean'][['implementation_cost', 'roi']]
+ lower_values = collapsed_data[collapsed_data['stat'] == 'lower'][['implementation_cost', 'roi']]
+ upper_values = collapsed_data[collapsed_data['stat'] == 'upper'][['implementation_cost', 'roi']]
+
+ # Plot mean line and confidence interval
+ ax.plot(
+ implementation_costs / 1e6,
+ mean_values['roi'],
+ label=f'{_scenario_dict[draw_index]}',
+ color=_draw_colors.get(draw_index, 'black'),
+ )
+ ax.fill_between(
+ implementation_costs / 1e6,
+ lower_values['roi'],
+ upper_values['roi'],
+ alpha=0.2,
+ color=_draw_colors.get(draw_index, 'black'),
+ )
+
+ max_val = mean_values[~np.isinf(mean_values['roi'])]['roi'].max()
+ max_roi.append(max_val)
+
+ # Capture ROI at specific costs
+ if _plot_vertical_lines_at:
+ for cost in _plot_vertical_lines_at:
+ roi_value = collapsed_data[
+ (collapsed_data.implementation_cost == cost) &
+ (collapsed_data.stat == 'mean')
+ ]['roi']
+ if not roi_value.empty:
+ roi_at_costs[cost].append(roi_value.iloc[0])
+
+ # Calculate and annotate ROI ratios
+ if _plot_vertical_lines_at:
+ for cost in _plot_vertical_lines_at:
+ if cost in roi_at_costs:
+ ratio = max(roi_at_costs[cost]) / min(roi_at_costs[cost])
+ ax.axvline(x=cost / 1e6, color='black', linestyle='--', linewidth=1)
+ ax.text(cost / 1e6 + ax.get_xlim()[1] * 0.011, ax.get_ylim()[1] * 0.75,
+ f'At {cost / 1e6:.0f}M, ratio of ROI curves = {round(ratio, 2)}',
+ color='black', fontsize=10, rotation=90, verticalalignment='top')
+
+ # Define fixed x-tick positions with a gap of 2000
+ step_size = (ax.get_xlim()[1] - 0)/5
+ xticks = np.arange(0, ax.get_xlim()[1] + 1, int(round(step_size, -3))) # From 0 to max x-limit with 5 steps
+ # Get labels
+ xtick_labels = [f'{tick:.0f}M' for tick in xticks] # Default labels for all ticks
+
+ # Replace specific x-ticks with % of health spending values
+ if _projected_health_spending:
+ xtick_labels[1] = f'{xticks[1]:.0f}M\n({xticks[1] / (_projected_health_spending / 1e6) :.2%} of \n projected total \n health spend)'
+ for i, tick in enumerate(xticks):
+ if (i != 0) & (i != 1): # Replace for 4000
+ xtick_labels[i] = f'{tick:.0f}M\n({tick / (_projected_health_spending/1e6) :.2%})'
+
+ # Update the x-ticks and labels
+ ax.set_xticks(xticks)
+ ax.set_xticklabels(xtick_labels, fontsize=10)
+
+ # Set y-axis limit
+ if _y_axis_lim == None:
+ ax.set_ylim(0, max(max_roi) * 1.25)
+ else:
+ ax.set_ylim(0, _y_axis_lim)
+ ax.set_xlim(left = 0)
+
+ plt.xlabel('Implementation cost, millions')
+ plt.ylabel('Return on Investment')
+ plt.title(f'Return on Investment at different levels of implementation cost{_year_suffix}')
+
+ # Show legend
+ plt.legend()
+ # Save
+ plt.savefig(_outputfilepath / f'draws_{_draws}_ROI_at_{_value_of_life_suffix}_{_year_suffix}.png', dpi=100,
+ bbox_inches='tight')
+ plt.close()
+
+'''
+# Scratch pad
+# TODO all these HR plots need to be looked at
+# 1. HR
+# Stacked bar chart of salaries by cadre
+def get_level_and_cadre_from_concatenated_value(_df, varname):
+ _df['Cadre'] = _df[varname].str.extract(r'=(.*?)\|')
+ _df['Facility_Level'] = _df[varname].str.extract(r'^[^=]*=[^|]*\|[^=]*=([^|]*)')
+ return _df
+def plot_cost_by_cadre_and_level(_df, figname_prefix, figname_suffix, draw):
+ if ('Facility_Level' in _df.columns) & ('Cadre' in _df.columns):
+ pass
+ else:
+ _df = get_level_and_cadre_from_concatenated_value(_df, 'OfficerType_FacilityLevel')
+
+ _df = _df[_df.draw == draw]
+ pivot_df = _df.pivot_table(index='Cadre', columns='Facility_Level', values='Cost',
+ aggfunc='sum', fill_value=0)
+ total_salary = round(_df['Cost'].sum(), 0)
+ total_salary = f"{total_salary:,.0f}"
+ ax = pivot_df.plot(kind='bar', stacked=True, title='Stacked Bar Graph by Cadre and Facility Level')
+ plt.ylabel(f'US Dollars')
+ plt.title(f"Annual {figname_prefix} cost by cadre and facility level")
+ plt.xticks(rotation=45)
+ plt.yticks(rotation=0)
+ plt.text(x=0.3, y=-0.5, s=f"Total {figname_prefix} cost = USD {total_salary}", transform=ax.transAxes,
+ horizontalalignment='center', fontsize=12, weight='bold', color='black')
+ plt.savefig(figurespath / f'{figname_prefix}_by_cadre_and_level_{figname_suffix}{draw}.png', dpi=100,
+ bbox_inches='tight')
+ plt.close()
+
+plot_cost_by_cadre_and_level(salary_for_all_staff,figname_prefix = "salary", figname_suffix= f"all_staff_draw", draw = 0)
+plot_cost_by_cadre_and_level(salary_for_staff_used_in_scenario.reset_index(),figname_prefix = "salary", figname_suffix= "staff_used_in_scenario_draw", draw = 0)
+plot_cost_by_cadre_and_level(recruitment_cost, figname_prefix = "recruitment", figname_suffix= "all_staff")
+plot_cost_by_cadre_and_level(preservice_training_cost, figname_prefix = "pre-service training", figname_suffix= "all_staff")
+plot_cost_by_cadre_and_level(inservice_training_cost, figname_prefix = "in-service training", figname_suffix= "all_staff")
+
+def plot_components_of_cost_category(_df, cost_category, figname_suffix):
+ pivot_df = _df[_df['Cost_Category'] == cost_category].pivot_table(index='Cost_Sub-category', values='Cost',
+ aggfunc='sum', fill_value=0)
+ ax = pivot_df.plot(kind='bar', stacked=False, title='Scenario Cost by Category')
+ plt.ylabel(f'US Dollars')
+ plt.title(f"Annual {cost_category} cost")
+ plt.xticks(rotation=45)
+ plt.yticks(rotation=0)
+
+ # Add text labels on the bars
+ total_cost = pivot_df['Cost'].sum()
+ rects = ax.patches
+ for rect, cost in zip(rects, pivot_df['Cost']):
+ cost_millions = cost / 1e6
+ percentage = (cost / total_cost) * 100
+ label_text = f"{cost_millions:.1f}M ({percentage:.1f}%)"
+ # Place text at the top of the bar
+ x = rect.get_x() + rect.get_width() / 2
+ y = rect.get_height()
+ ax.text(x, y, label_text, ha='center', va='bottom', fontsize=8, rotation=0)
+
+ total_cost = f"{total_cost:,.0f}"
+ plt.text(x=0.3, y=-0.5, s=f"Total {cost_category} cost = USD {total_cost}", transform=ax.transAxes,
+ horizontalalignment='center', fontsize=12, weight='bold', color='black')
+
+ plt.savefig(figurespath / f'{cost_category}_by_cadre_and_level_{figname_suffix}.png', dpi=100,
+ bbox_inches='tight')
+ plt.close()
+
+plot_components_of_cost_category(_df = scenario_cost, cost_category = 'Human Resources for Health', figname_suffix = "all_staff")
+
+
+# Compare financial costs with actual budget data
+####################################################
+# Import budget data
+budget_data = workbook_cost["budget_validation"]
+list_of_costs_for_comparison = ['total_salary_for_all_staff', 'total_cost_of_consumables_dispensed', 'total_cost_of_consumables_stocked']
+real_budget = [budget_data[budget_data['Category'] == list_of_costs_for_comparison[0]]['Budget_in_2023USD'].values[0],
+ budget_data[budget_data['Category'] == list_of_costs_for_comparison[1]]['Budget_in_2023USD'].values[0],
+ budget_data[budget_data['Category'] == list_of_costs_for_comparison[1]]['Budget_in_2023USD'].values[0]]
+model_cost = [scenario_cost_financial[scenario_cost_financial['Cost_Sub-category'] == list_of_costs_for_comparison[0]]['Value_2023USD'].values[0],
+ scenario_cost_financial[scenario_cost_financial['Cost_Sub-category'] == list_of_costs_for_comparison[1]]['Value_2023USD'].values[0],
+ scenario_cost_financial[scenario_cost_financial['Cost_Sub-category'] == list_of_costs_for_comparison[2]]['Value_2023USD'].values[0]]
+
+plt.clf()
+plt.scatter(real_budget, model_cost)
+# Plot a line representing a 45-degree angle
+min_val = min(min(real_budget), min(model_cost))
+max_val = max(max(real_budget), max(model_cost))
+plt.plot([min_val, max_val], [min_val, max_val], 'r--', label='45-degree line')
+
+# Format x and y axis labels to display in millions
+formatter = FuncFormatter(lambda x, _: '{:,.0f}M'.format(x / 1e6))
+plt.gca().xaxis.set_major_formatter(formatter)
+plt.gca().yaxis.set_major_formatter(formatter)
+# Add labels for each point
+hr_label = 'HR_salary ' + f'{round(model_cost[0] / real_budget[0], 2)}'
+consumables_label1= 'Consumables dispensed ' + f'{round(model_cost[1] / real_budget[1], 2)}'
+consumables_label2 = 'Consumables stocked ' + f'{round(model_cost[2] / real_budget[2], 2)}'
+plotlabels = [hr_label, consumables_label1, consumables_label2]
+for i, txt in enumerate(plotlabels):
+ plt.text(real_budget[i], model_cost[i], txt, ha='right')
+
+plt.xlabel('Real Budget')
+plt.ylabel('Model Cost')
+plt.title('Real Budget vs Model Cost')
+plt.savefig(costing_outputs_folder / 'Cost_validation.png')
+
+# Explore the ratio of consumable inflows to outflows
+######################################################
+# TODO: Only consider the months for which original OpenLMIS data was available for closing_stock and dispensed
+def plot_inflow_to_outflow_ratio(_dict, groupby_var):
+ # Convert Dict to dataframe
+ flattened_data = [(level1, level2, level3, level4, value) for (level1, level2, level3, level4), value in
+ inflow_to_outflow_ratio.items()] # Flatten dictionary into a list of tuples
+ _df = pd.DataFrame(flattened_data, columns=['category', 'item_code', 'district', 'fac_type_tlo', 'inflow_to_outflow_ratio']) # Convert flattened data to DataFrame
+
+ # Plot the bar plot
+ plt.figure(figsize=(10, 6))
+ sns.barplot(data=_df , x=groupby_var, y= 'inflow_to_outflow_ratio', errorbar=None)
+
+ # Add points representing the distribution of individual values
+ sns.stripplot(data=_df, x=groupby_var, y='inflow_to_outflow_ratio', color='black', size=5, alpha=0.2)
+
+ # Set labels and title
+ plt.xlabel(groupby_var)
+ plt.ylabel('Inflow to Outflow Ratio')
+ plt.title('Average Inflow to Outflow Ratio by ' + f'{groupby_var}')
+ plt.xticks(rotation=45)
+
+ # Show plot
+ plt.tight_layout()
+ plt.savefig(costing_outputs_folder / 'inflow_to_outflow_ratio_by' f'{groupby_var}' )
+
+plot_inflow_to_outflow_ratio(inflow_to_outflow_ratio, 'fac_type_tlo')
+plot_inflow_to_outflow_ratio(inflow_to_outflow_ratio, 'district')
+plot_inflow_to_outflow_ratio(inflow_to_outflow_ratio, 'item_code')
+plot_inflow_to_outflow_ratio(inflow_to_outflow_ratio, 'category')
+
+# Plot fraction staff time used
+fraction_stafftime_average = salary_staffneeded_df.groupby('Officer_Category')['Value'].sum()
+fraction_stafftime_average. plot(kind = "bar")
+plt.xlabel('Cadre')
+plt.ylabel('Fraction time needed')
+plt.savefig(costing_outputs_folder / 'hr_time_need_economic_cost.png')
+
+# Plot salary costs by cadre and facility level
+# Group by cadre and level
+salary_for_all_staff[['Officer_Type', 'Facility_Level']] = salary_for_all_staff['OfficerType_FacilityLevel'].str.split('|', expand=True)
+salary_for_all_staff['Officer_Type'] = salary_for_all_staff['Officer_Type'].str.replace('Officer_Type=', '')
+salary_for_all_staff['Facility_Level'] = salary_for_all_staff['Facility_Level'].str.replace('Facility_Level=', '')
+total_salary_by_cadre = salary_for_all_staff.groupby('Officer_Type')['Total_salary_by_cadre_and_level'].sum()
+total_salary_by_level = salary_for_all_staff.groupby('Facility_Level')['Total_salary_by_cadre_and_level'].sum()
+
+# Plot by cadre
+plt.clf()
+total_salary_by_cadre.plot(kind='bar')
+plt.xlabel('Officer_category')
+plt.ylabel('Total Salary')
+plt.title('Total Salary by Cadre')
+plt.savefig(costing_outputs_folder / 'total_salary_by_cadre.png')
+
+# Plot by level
+plt.clf()
+total_salary_by_level.plot(kind='bar')
+plt.xlabel('Facility_Level')
+plt.ylabel('Total Salary')
+plt.title('Total Salary by Facility_Level')
+plt.savefig(costing_outputs_folder / 'total_salary_by_level.png')
+
+
+log['tlo.methods.healthsystem']['Capacity']['Frac_Time_Used_By_Facility_ID'] # for district disaggregation
+
+# Aggregate Daily capabilities to total used by cadre and facility level
+
+# log['tlo.methods.healthsystem.summary']['Capacity']['Frac_Time_Used_By_OfficerType']
+# 1.2 HR cost by Treatment_ID
+# For HR cost by Treatment_ID, multiply total cost by Officer type by fraction of time used for treatment_ID
+log['tlo.methods.healthsystem.summary']['HSI_Event']['TREATMENT_ID'] # what does this represent? why are there 3 rows (2 scenarios)
+# But what we need is the HR use by Treatment_ID - Leave this for later?
+
+# log['tlo.scenario']
+log['tlo.methods.healthsystem.summary']['HSI_Event']['Number_By_Appt_Type_Code']
+
+
+df = pd.DataFrame(log['tlo.methods.healthsystem.summary'])
+df.to_csv(outputfilepath / 'temp.csv')
+
+def read_parameters(self, data_folder):
+ """
+ 1. Reads the costing resource file
+ 2. Declares the costing parameters
+ """
+ # Read the resourcefile
+ # Short cut to parameters dict
+ p = self.parameters
+
+ workbook = pd.read_excel((resourcefilepath / "ResourceFile_Costing.xlsx"),
+ sheet_name = None)
+
+ p["human_resources"] = workbook["human_resources"]
+
+workbook = pd.read_excel((resourcefilepath / "ResourceFile_Costing.xlsx"),
+ sheet_name = None)
+human_resources = workbook["human_resources"]
+
+'''
+
+'''
+consumables_dispensed_under_perfect_availability = get_quantity_of_consumables_dispensed(consumables_results_folder)[9]
+consumables_dispensed_under_perfect_availability = consumables_dispensed_under_perfect_availability['mean'].to_dict() # TODO incorporate uncertainty in estimates
+consumables_dispensed_under_perfect_availability = defaultdict(int, {int(key): value for key, value in
+ consumables_dispensed_under_perfect_availability.items()}) # Convert string keys to integer
+consumables_dispensed_under_default_availability = get_quantity_of_consumables_dispensed(consumables_results_folder)[0]
+consumables_dispensed_under_default_availability = consumables_dispensed_under_default_availability['mean'].to_dict()
+consumables_dispensed_under_default_availability = defaultdict(int, {int(key): value for key, value in
+ consumables_dispensed_under_default_availability.items()}) # Convert string keys to integer
+
+# Load consumables cost data
+unit_price_consumable = workbook_cost["consumables"]
+unit_price_consumable = unit_price_consumable.rename(columns=unit_price_consumable.iloc[0])
+unit_price_consumable = unit_price_consumable[['Item_Code', 'Final_price_per_chosen_unit (USD, 2023)']].reset_index(drop=True).iloc[1:]
+unit_price_consumable = unit_price_consumable[unit_price_consumable['Item_Code'].notna()]
+unit_price_consumable = unit_price_consumable.set_index('Item_Code').to_dict(orient='index')
+
+# 2.1 Cost of consumables dispensed
+#---------------------------------------------------------------------------------------------------------------
+# Multiply number of items needed by cost of consumable
+cost_of_consumables_dispensed_under_perfect_availability = {key: unit_price_consumable[key]['Final_price_per_chosen_unit (USD, 2023)'] * consumables_dispensed_under_perfect_availability[key] for
+ key in unit_price_consumable if key in consumables_dispensed_under_perfect_availability}
+total_cost_of_consumables_dispensed_under_perfect_availability = sum(value for value in cost_of_consumables_dispensed_under_perfect_availability.values() if not np.isnan(value))
+
+cost_of_consumables_dispensed_under_default_availability = {key: unit_price_consumable[key]['Final_price_per_chosen_unit (USD, 2023)'] * consumables_dispensed_under_default_availability[key] for
+ key in unit_price_consumable if key in consumables_dispensed_under_default_availability}
+total_cost_of_consumables_dispensed_under_default_availability = sum(value for value in cost_of_consumables_dispensed_under_default_availability.values() if not np.isnan(value))
+def convert_dict_to_dataframe(_dict):
+ data = {key: [value] for key, value in _dict.items()}
+ _df = pd.DataFrame(data)
+ return _df
+
+cost_perfect_df = convert_dict_to_dataframe(cost_of_consumables_dispensed_under_perfect_availability).T.rename(columns = {0:"cost_dispensed_stock_perfect_availability"}).round(2)
+cost_default_df = convert_dict_to_dataframe(cost_of_consumables_dispensed_under_default_availability).T.rename(columns = {0:"cost_dispensed_stock_default_availability"}).round(2)
+unit_cost_df = convert_dict_to_dataframe(unit_price_consumable).T.rename(columns = {0:"unit_cost"})
+dispensed_default_df = convert_dict_to_dataframe(consumables_dispensed_under_default_availability).T.rename(columns = {0:"dispensed_default_availability"}).round(2)
+dispensed_perfect_df = convert_dict_to_dataframe(consumables_dispensed_under_perfect_availability).T.rename(columns = {0:"dispensed_perfect_availability"}).round(2)
+
+full_cons_cost_df = pd.merge(cost_perfect_df, cost_default_df, left_index=True, right_index=True)
+full_cons_cost_df = pd.merge(full_cons_cost_df, unit_cost_df, left_index=True, right_index=True)
+full_cons_cost_df = pd.merge(full_cons_cost_df, dispensed_default_df, left_index=True, right_index=True)
+full_cons_cost_df = pd.merge(full_cons_cost_df, dispensed_perfect_df, left_index=True, right_index=True)
+
+# 2.2 Cost of consumables stocked (quantity needed for what is dispensed)
+#---------------------------------------------------------------------------------------------------------------
+# Stocked amount should be higher than dispensed because of i. excess capacity, ii. theft, iii. expiry
+# While there are estimates in the literature of what % these might be, we agreed that it is better to rely upon
+# an empirical estimate based on OpenLMIS data
+# Estimate the stock to dispensed ratio from OpenLMIS data
+lmis_consumable_usage = pd.read_csv(path_for_consumable_resourcefiles / "ResourceFile_Consumables_availability_and_usage.csv")
+# Collapse individual facilities
+lmis_consumable_usage_by_item_level_month = lmis_consumable_usage.groupby(['category', 'item_code', 'district', 'fac_type_tlo', 'month'])[['closing_bal', 'dispensed', 'received']].sum()
+df = lmis_consumable_usage_by_item_level_month # Drop rows where monthly OpenLMIS data wasn't available
+df = df.loc[df.index.get_level_values('month') != "Aggregate"]
+opening_bal_january = df.loc[df.index.get_level_values('month') == 'January', 'closing_bal'] + \
+ df.loc[df.index.get_level_values('month') == 'January', 'dispensed'] - \
+ df.loc[df.index.get_level_values('month') == 'January', 'received']
+closing_bal_december = df.loc[df.index.get_level_values('month') == 'December', 'closing_bal']
+total_consumables_inflow_during_the_year = df.loc[df.index.get_level_values('month') != 'January', 'received'].groupby(level=[0,1,2,3]).sum() +\
+ opening_bal_january.reset_index(level='month', drop=True) -\
+ closing_bal_december.reset_index(level='month', drop=True)
+total_consumables_outflow_during_the_year = df['dispensed'].groupby(level=[0,1,2,3]).sum()
+inflow_to_outflow_ratio = total_consumables_inflow_during_the_year.div(total_consumables_outflow_during_the_year, fill_value=1)
+
+# Edit outlier ratios
+inflow_to_outflow_ratio.loc[inflow_to_outflow_ratio < 1] = 1 # Ratio can't be less than 1
+inflow_to_outflow_ratio.loc[inflow_to_outflow_ratio > inflow_to_outflow_ratio.quantile(0.95)] = inflow_to_outflow_ratio.quantile(0.95) # Trim values greater than the 95th percentile
+average_inflow_to_outflow_ratio_ratio = inflow_to_outflow_ratio.mean()
+#inflow_to_outflow_ratio.loc[inflow_to_outflow_ratio.isna()] = average_inflow_to_outflow_ratio_ratio # replace missing with average
+
+# Multiply number of items needed by cost of consumable
+inflow_to_outflow_ratio_by_consumable = inflow_to_outflow_ratio.groupby(level='item_code').mean()
+excess_stock_ratio = inflow_to_outflow_ratio_by_consumable - 1
+excess_stock_ratio = excess_stock_ratio.to_dict()
+# TODO Consider whether a more disaggregated version of the ratio dictionary should be applied
+cost_of_excess_consumables_stocked_under_perfect_availability = dict(zip(unit_price_consumable, (unit_price_consumable[key]['Final_price_per_chosen_unit (USD, 2023)'] *
+ consumables_dispensed_under_perfect_availability[key] *
+ excess_stock_ratio.get(key, average_inflow_to_outflow_ratio_ratio - 1)
+ for key in consumables_dispensed_under_perfect_availability)))
+cost_of_excess_consumables_stocked_under_default_availability = dict(zip(unit_price_consumable, (unit_price_consumable[key]['Final_price_per_chosen_unit (USD, 2023)'] *
+ consumables_dispensed_under_default_availability[key] *
+ excess_stock_ratio.get(key, average_inflow_to_outflow_ratio_ratio - 1)
+ for key in consumables_dispensed_under_default_availability)))
+cost_excess_stock_perfect_df = convert_dict_to_dataframe(cost_of_excess_consumables_stocked_under_perfect_availability).T.rename(columns = {0:"cost_excess_stock_perfect_availability"}).round(2)
+cost_excess_stock_default_df = convert_dict_to_dataframe(cost_of_excess_consumables_stocked_under_default_availability).T.rename(columns = {0:"cost_excess_stock_default_availability"}).round(2)
+full_cons_cost_df = pd.merge(full_cons_cost_df, cost_excess_stock_perfect_df, left_index=True, right_index=True)
+full_cons_cost_df = pd.merge(full_cons_cost_df, cost_excess_stock_default_df, left_index=True, right_index=True)
+
+total_cost_of_excess_consumables_stocked_under_perfect_availability = sum(value for value in cost_of_excess_consumables_stocked_under_perfect_availability.values() if not np.isnan(value))
+total_cost_of_excess_consumables_stocked_under_default_availability = sum(value for value in cost_of_excess_consumables_stocked_under_default_availability.values() if not np.isnan(value))
+
+full_cons_cost_df = full_cons_cost_df.reset_index().rename(columns = {'index' : 'item_code'})
+full_cons_cost_df.to_csv(figurespath / 'consumables_cost_220824.csv')
+
+# Import data for plotting
+tlo_lmis_mapping = pd.read_csv(path_for_consumable_resourcefiles / 'ResourceFile_consumables_matched.csv', low_memory=False, encoding="ISO-8859-1")[['item_code', 'module_name', 'consumable_name_tlo']]
+tlo_lmis_mapping = tlo_lmis_mapping[~tlo_lmis_mapping['item_code'].duplicated(keep='first')]
+full_cons_cost_df = pd.merge(full_cons_cost_df, tlo_lmis_mapping, on = 'item_code', how = 'left', validate = "1:1")
+full_cons_cost_df['total_cost_perfect_availability'] = full_cons_cost_df['cost_dispensed_stock_perfect_availability'] + full_cons_cost_df['cost_excess_stock_perfect_availability']
+full_cons_cost_df['total_cost_default_availability'] = full_cons_cost_df['cost_dispensed_stock_default_availability'] + full_cons_cost_df['cost_excess_stock_default_availability']
+
+def recategorize_modules_into_consumable_categories(_df):
+ _df['category'] = _df['module_name'].str.lower()
+ cond_RH = (_df['category'].str.contains('care_of_women_during_pregnancy')) | \
+ (_df['category'].str.contains('labour'))
+ cond_newborn = (_df['category'].str.contains('newborn'))
+ cond_newborn[cond_newborn.isna()] = False
+ cond_childhood = (_df['category'] == 'acute lower respiratory infections') | \
+ (_df['category'] == 'measles') | \
+ (_df['category'] == 'diarrhoea')
+ cond_rti = _df['category'] == 'road traffic injuries'
+ cond_cancer = _df['category'].str.contains('cancer')
+ cond_cancer[cond_cancer.isna()] = False
+ cond_ncds = (_df['category'] == 'epilepsy') | \
+ (_df['category'] == 'depression')
+ _df.loc[cond_RH, 'category'] = 'reproductive_health'
+ _df.loc[cond_cancer, 'category'] = 'cancer'
+ _df.loc[cond_newborn, 'category'] = 'neonatal_health'
+ _df.loc[cond_childhood, 'category'] = 'other_childhood_illnesses'
+ _df.loc[cond_rti, 'category'] = 'road_traffic_injuries'
+ _df.loc[cond_ncds, 'category'] = 'ncds'
+ cond_condom = _df['item_code'] == 2
+ _df.loc[cond_condom, 'category'] = 'contraception'
+
+ # Create a general consumables category
+ general_cons_list = [300, 33, 57, 58, 141, 5, 6, 10, 21, 23, 127, 24, 80, 93, 144, 149, 154, 40, 67, 73, 76,
+ 82, 101, 103, 88, 126, 135, 71, 98, 171, 133, 134, 244, 247, 49, 112, 1933, 1960]
+ cond_general = _df['item_code'].isin(general_cons_list)
+ _df.loc[cond_general, 'category'] = 'general'
+
+ return _df
+
+full_cons_cost_df = recategorize_modules_into_consumable_categories(full_cons_cost_df)
+# Fill gaps in categories
+dict_for_missing_categories = {292: 'acute lower respiratory infections', 293: 'acute lower respiratory infections',
+ 307: 'reproductive_health', 2019: 'reproductive_health',
+ 2678: 'tb', 1171: 'other_childhood_illnesses', 1237: 'cancer', 1239: 'cancer'}
+# Use map to create a new series from item_code to fill missing values in category
+mapped_categories = full_cons_cost_df['item_code'].map(dict_for_missing_categories)
+# Use fillna on the 'category' column to fill missing values using the mapped_categories
+full_cons_cost_df['category'] = full_cons_cost_df['category'].fillna(mapped_categories)
+
+# Bar plot of cost of dispensed consumables
+def plot_consumable_cost(_df, suffix, groupby_var, top_x_values = float('nan')):
+ pivot_df = _df.groupby(groupby_var)['cost_' + suffix].sum().reset_index()
+ pivot_df['cost_' + suffix] = pivot_df['cost_' + suffix]/1e6
+ if math.isnan(top_x_values):
+ pass
+ else:
+ pivot_df = pivot_df.sort_values('cost_' + suffix, ascending = False)[1:top_x_values]
+ total_cost = round(_df['cost_' + suffix].sum(), 0)
+ total_cost = f"{total_cost:,.0f}"
+ ax = pivot_df['cost_' + suffix].plot(kind='bar', stacked=False, title=f'Consumables cost by {groupby_var}')
+ # Setting x-ticks explicitly
+ #ax.set_xticks(range(len(pivot_df['category'])))
+ ax.set_xticklabels(pivot_df[groupby_var], rotation=45)
+ plt.ylabel(f'US Dollars (millions)')
+ plt.title(f"Annual consumables cost by {groupby_var} (assuming {suffix})")
+ plt.xticks(rotation=90)
+ plt.yticks(rotation=0)
+ plt.text(x=0.5, y=-0.8, s=f"Total consumables cost =\n USD {total_cost}", transform=ax.transAxes,
+ horizontalalignment='center', fontsize=12, weight='bold', color='black')
+ plt.savefig(figurespath / f'consumables_cost_by_{groupby_var}_{suffix}.png', dpi=100,
+ bbox_inches='tight')
+ plt.close()
+
+plot_consumable_cost(_df = full_cons_cost_df,suffix = 'dispensed_stock_perfect_availability', groupby_var = 'category')
+plot_consumable_cost(_df = full_cons_cost_df, suffix = 'dispensed_stock_default_availability', groupby_var = 'category')
+
+# Plot the 10 consumables with the highest cost
+plot_consumable_cost(_df = full_cons_cost_df,suffix = 'dispensed_stock_perfect_availability', groupby_var = 'consumable_name_tlo', top_x_values = 10)
+plot_consumable_cost(_df = full_cons_cost_df,suffix = 'dispensed_stock_default_availability', groupby_var = 'consumable_name_tlo', top_x_values = 10)
+
+def plot_cost_by_category(_df, suffix , figname_prefix = 'Consumables'):
+ pivot_df = full_cons_cost_df[['category', 'cost_dispensed_stock_' + suffix, 'cost_excess_stock_' + suffix]]
+ pivot_df = pivot_df.groupby('category')[['cost_dispensed_stock_' + suffix, 'cost_excess_stock_' + suffix]].sum()
+ total_cost = round(_df['total_cost_' + suffix].sum(), 0)
+ total_cost = f"{total_cost:,.0f}"
+ ax = pivot_df.plot(kind='bar', stacked=True, title='Stacked Bar Graph by Category')
+ plt.ylabel(f'US Dollars')
+ plt.title(f"Annual {figname_prefix} cost by category")
+ plt.xticks(rotation=90, size = 9)
+ plt.yticks(rotation=0)
+ plt.text(x=0.3, y=-0.5, s=f"Total {figname_prefix} cost = USD {total_cost}", transform=ax.transAxes,
+ horizontalalignment='center', fontsize=12, weight='bold', color='black')
+ plt.savefig(figurespath / f'{figname_prefix}_by_category_{suffix}.png', dpi=100,
+ bbox_inches='tight')
+ plt.close()
+
+plot_cost_by_category(full_cons_cost_df, suffix = 'perfect_availability' , figname_prefix = 'Consumables')
+plot_cost_by_category(full_cons_cost_df, suffix = 'default_availability' , figname_prefix = 'Consumables')
+'''
+
+'''
+# Plot equipment cost
+# Plot different categories of cost by level of care
+def plot_components_of_cost_category(_df, cost_category, figname_suffix):
+ pivot_df = _df[_df['Cost_Category'] == cost_category].pivot_table(index='Cost_Sub-category', values='Cost',
+ aggfunc='sum', fill_value=0)
+ ax = pivot_df.plot(kind='bar', stacked=False, title='Scenario Cost by Category')
+ plt.ylabel(f'US Dollars')
+ plt.title(f"Annual {cost_category} cost")
+ plt.xticks(rotation=45)
+ plt.yticks(rotation=0)
+
+ # Add text labels on the bars
+ total_cost = pivot_df['Cost'].sum()
+ rects = ax.patches
+ for rect, cost in zip(rects, pivot_df['Cost']):
+ cost_millions = cost / 1e6
+ percentage = (cost / total_cost) * 100
+ label_text = f"{cost_millions:.1f}M ({percentage:.1f}%)"
+ # Place text at the top of the bar
+ x = rect.get_x() + rect.get_width() / 2
+ y = rect.get_height()
+ ax.text(x, y, label_text, ha='center', va='bottom', fontsize=8, rotation=0)
+
+ total_cost = f"{total_cost:,.0f}"
+ plt.text(x=0.3, y=-0.5, s=f"Total {cost_category} cost = USD {total_cost}", transform=ax.transAxes,
+ horizontalalignment='center', fontsize=12, weight='bold', color='black')
+
+ plt.savefig(figurespath / f'{cost_category}_{figname_suffix}.png', dpi=100,
+ bbox_inches='tight')
+ plt.close()
+
+plot_components_of_cost_category(_df = scenario_cost, cost_category = 'Equipment', figname_suffix = "")
+
+# Plot top 10 most expensive items
+def plot_most_expensive_equipment(_df, top_x_values = 10, figname_prefix = "Equipment"):
+ top_x_items = _df.groupby('Item_code')['annual_cost'].sum().sort_values(ascending = False)[0:top_x_values-1].index
+ _df_subset = _df[_df.Item_code.isin(top_x_items)]
+
+ pivot_df = _df_subset.pivot_table(index='Equipment_tlo', columns='Facility_Level', values='annual_cost',
+ aggfunc='sum', fill_value=0)
+ ax = pivot_df.plot(kind='bar', stacked=True, title='Stacked Bar Graph by Item and Facility Level')
+ plt.ylabel(f'US Dollars')
+ plt.title(f"Annual {figname_prefix} cost by item and facility level")
+ plt.xticks(rotation=90, size = 8)
+ plt.yticks(rotation=0)
+ plt.savefig(figurespath / f'{figname_prefix}_by_item_and_level.png', dpi=100,
+ bbox_inches='tight')
+ plt.close()
+
+
+plot_most_expensive_equipment(equipment_cost)
+
+# TODO PLot which equipment is used by district and facility or a heatmap of the number of facilities at which an equipment is used
+# TODO Collapse facility IDs by level of care to get the total number of facilities at each level using an item
+# TODO Multiply number of facilities by level with the quantity needed of each equipment and collapse to get total number of equipment (nationally)
+# TODO Which equipment needs to be newly purchased (currently no assumption made for equipment with cost > $250,000)
+
+'''
diff --git a/src/scripts/costing/costing_overview_analysis.py b/src/scripts/costing/costing_overview_analysis.py
new file mode 100644
index 0000000000..a4f1bc24db
--- /dev/null
+++ b/src/scripts/costing/costing_overview_analysis.py
@@ -0,0 +1,347 @@
+"""Produce outputs for cost overview paper.
+The draft version of the paper uses outputs from scenario_impact_of_healthsystem.py, used to model HSS scenarios for
+FCDO and Global Fund.
+
+with reduced consumables logging
+/Users/tmangal/PycharmProjects/TLOmodel/outputs/t.mangal@imperial.ac.uk/hss_elements-2024-11-12T172311Z
+"""
+
+from pathlib import Path
+from tlo import Date
+
+import datetime
+import os
+import textwrap
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+import squarify
+import numpy as np
+import pandas as pd
+from itertools import cycle
+
+from tlo.analysis.utils import (
+ extract_params,
+ extract_results,
+ get_scenario_info,
+ get_scenario_outputs,
+ load_pickled_dataframes,
+ summarize
+)
+
+from scripts.costing.cost_estimation import (estimate_input_cost_of_scenarios,
+ summarize_cost_data,
+ do_stacked_bar_plot_of_cost_by_category,
+ do_line_plot_of_cost,
+ create_summary_treemap_by_cost_subgroup,
+ estimate_projected_health_spending)
+
+# Define a timestamp for script outputs
+timestamp = datetime.datetime.now().strftime("_%Y_%m_%d_%H_%M")
+
+# Print the start time of the script
+print('Script Start', datetime.datetime.now().strftime('%H:%M'))
+
+# Create folders to store results
+resourcefilepath = Path("./resources")
+outputfilepath = Path('./outputs/t.mangal@imperial.ac.uk')
+figurespath = Path('./outputs/costing/overview/')
+if not os.path.exists(figurespath):
+ os.makedirs(figurespath)
+
+# Load result files
+# ------------------------------------------------------------------------------------------------------------------
+results_folder = get_scenario_outputs('hss_elements-2024-11-12T172311Z.py', outputfilepath)[0]
+#results_folder = Path('./outputs/cost_scenarios-2024-11-26T205921Z')
+
+# Check can read results from draw=0, run=0
+log = load_pickled_dataframes(results_folder, 0, 0) # look at one log (so can decide what to extract)
+params = extract_params(results_folder)
+
+# Declare default parameters for cost analysis
+# ------------------------------------------------------------------------------------------------------------------
+# Period relevant for costing
+TARGET_PERIOD = (Date(2010, 1, 1), Date(2030, 12, 31)) # This is the period that is costed
+relevant_period_for_costing = [i.year for i in TARGET_PERIOD]
+list_of_relevant_years_for_costing = list(range(relevant_period_for_costing[0], relevant_period_for_costing[1] + 1))
+list_of_years_for_plot = list(range(2023, 2031))
+number_of_years_costed = relevant_period_for_costing[1] - 2023 + 1
+
+# Scenarios
+cost_scenarios = {0: "Actual", 3: "Expanded HRH", 5: "Improved consumable availability",
+ 8: "Expanded HRH + Improved consumable availability"}
+
+# Costing parameters
+discount_rate = 0.03
+discount_rate_lomas = -0.0079
+
+# Estimate standard input costs of scenario
+# -----------------------------------------------------------------------------------------------------------------------
+input_costs = estimate_input_cost_of_scenarios(results_folder, resourcefilepath, _draws = [0, 3, 5, 8],
+ _years=list_of_relevant_years_for_costing, cost_only_used_staff=True,
+ _discount_rate = discount_rate, summarize = True)
+input_costs = input_costs[(input_costs.year > 2022) & (input_costs.year < 2031)]
+# _draws = htm_scenarios_for_gf_report --> this subset is created after calculating malaria scale up costs
+
+input_costs_undiscounted = estimate_input_cost_of_scenarios(results_folder, resourcefilepath, _draws = [0, 3, 5, 8],
+ _years=list_of_relevant_years_for_costing, cost_only_used_staff=True,
+ _discount_rate = 0, summarize = True)
+input_costs_undiscounted = input_costs_undiscounted[(input_costs_undiscounted.year > 2022) & (input_costs_undiscounted.year < 2031)]
+
+input_costs_variable_discounting = estimate_input_cost_of_scenarios(results_folder, resourcefilepath, _draws = [0, 3, 5, 8],
+ _years=list_of_relevant_years_for_costing, cost_only_used_staff=True,
+ _discount_rate = discount_rate_lomas, summarize = True)
+input_costs_variable_discounting = input_costs_variable_discounting[(input_costs_variable_discounting.year > 2022) & (input_costs_variable_discounting.year < 2031)]
+
+# _draws = htm_scenarios_for_gf_report --> this subset is created after calculating malaria scale up costs
+
+# Get overall estimates for main text
+# -----------------------------------------------------------------------------------------------------------------------
+cost_by_draw = input_costs.groupby(['draw', 'stat'])['cost'].sum()
+print(f"The total cost of healthcare delivery in Malawi between 2023 and 2030 was estimated to be "
+ f"\${cost_by_draw[0,'mean']/1e9:,.2f} billion [95\% confidence interval (CI), \${cost_by_draw[0,'lower']/1e9:,.2f}b - \${cost_by_draw[0,'upper']/1e9:,.2f}b], under the actual scenario, and increased to "
+ f"\${cost_by_draw[5,'mean']/1e9:,.2f} billion [\${cost_by_draw[5,'lower']/1e9:,.2f}b - \${cost_by_draw[5,'upper']/1e9:,.2f}b] under the improved consumable availability scenario, "
+ f"followed by \${cost_by_draw[3,'mean']/1e9:,.2f} billion [\${cost_by_draw[3,'lower']/1e9:,.2f}b - \${cost_by_draw[3,'upper']/1e9:,.2f}b] under the expanded HRH scenario and finally "
+ f"\${cost_by_draw[8,'mean']/1e9:,.2f} billion [\${cost_by_draw[8,'lower']/1e9:,.2f}b - \${cost_by_draw[8,'upper']/1e9:,.2f}b] under the expanded HRH + improved consumable availability scenario.")
+
+undiscounted_cost_by_draw = input_costs_undiscounted.groupby(['draw', 'stat'])['cost'].sum()
+print(f"The average annual cost of healthcare delivery in Malawi between 2023 and 2030 was estimated to be "
+ f"\${undiscounted_cost_by_draw[0,'mean']/1e6/number_of_years_costed:,.2f} million [\${undiscounted_cost_by_draw[0,'lower']/1e6/number_of_years_costed:,.2f}m - \${undiscounted_cost_by_draw[0,'upper']/1e6/number_of_years_costed:,.2f}m], under the actual scenario, and increased to "
+ f"\${undiscounted_cost_by_draw[5,'mean']/1e6/number_of_years_costed:,.2f} million [\${undiscounted_cost_by_draw[5,'lower']/1e6/number_of_years_costed:,.2f}m - \${undiscounted_cost_by_draw[5,'upper']/1e6/number_of_years_costed:,.2f}m] under the improved consumable availability scenario, "
+ f"followed by \${undiscounted_cost_by_draw[3,'mean']/1e6/number_of_years_costed:,.2f} million [\${undiscounted_cost_by_draw[3,'lower']/1e6/number_of_years_costed:,.2f}m - \${undiscounted_cost_by_draw[3,'upper']/1e6/number_of_years_costed:,.2f}m] under the expanded HRH scenario and finally "
+ f"\${undiscounted_cost_by_draw[8,'mean']/1e6/number_of_years_costed:,.2f} million [\${undiscounted_cost_by_draw[8,'lower']/1e6/number_of_years_costed:,.2f}m - \${undiscounted_cost_by_draw[8,'upper']/1e6/number_of_years_costed:,.2f}m] under the expanded HRH + improved consumable availability scenario.")
+
+print(f"Based on this method, the total cost of healthcare delivery in Malawi between 2023 and 2030 was estimated at "
+ f"\${cost_by_draw[0,'mean']/1e9:,.2f} billion[\${cost_by_draw[0,'lower']/1e9:,.2f}b - \${cost_by_draw[0,'upper']/1e9:,.2f}b]"
+ f"(average annual cost of \${undiscounted_cost_by_draw[0,'mean']/1e6/number_of_years_costed:,.2f} million[\${undiscounted_cost_by_draw[0,'lower']/1e6/number_of_years_costed:,.2f}m - \${undiscounted_cost_by_draw[0,'upper']/1e6/number_of_years_costed:,.2f}m] "
+ f"under current health system capacity. "
+ f"Alternative scenarios reflecting improvements in consuambles supply chain performance and workforce size increased costs by "
+ f"{(cost_by_draw[5,'mean']/cost_by_draw[0,'mean'] - 1):.2%} to "
+ f"{(cost_by_draw[8,'mean']/cost_by_draw[0,'mean'] - 1):.2%}. ")
+
+consumable_cost_by_draw = input_costs[(input_costs.cost_category == 'medical consumables') & (input_costs.stat == 'mean')].groupby(['draw'])['cost'].sum()
+print(f"Notably, we find that the improved consumable availability scenario results in a {(consumable_cost_by_draw[3]/consumable_cost_by_draw[0] - 1):.2%} "
+ f"increase in cost of medical consumables. However, when combined with expanded HRH, the increase in consumables dispensed is "
+ f"{(consumable_cost_by_draw[8]/consumable_cost_by_draw[0] - 1):.2%} more than the actual scenario because the health system is able to deliver more appointments.")
+
+cost_of_hiv_testing = input_costs[(input_costs.cost_subgroup == 'Test, HIV EIA Elisa') & (input_costs.stat == 'mean')].groupby(['draw'])['cost'].sum()
+print(f"For instance, the cost of HIV testing consumables increases by {(cost_of_hiv_testing[3]/cost_of_hiv_testing[0] - 1):.2%} under the expanded HRH scenario and by "
+ f"{(cost_of_hiv_testing[8]/cost_of_hiv_testing[0] - 1):.2%} under the combined expanded HRH and improved consumable availability scenario, "
+ f"while showing almost no change under the scenario with improved consumable availability alone")
+
+# Get figures for overview paper
+# -----------------------------------------------------------------------------------------------------------------------
+# Figure 1: Estimated costs by cost category
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs, _cost_category = 'all', _disaggregate_by_subgroup = False,
+ _year = list_of_years_for_plot,
+ _outputfilepath = figurespath, _scenario_dict = cost_scenarios)
+
+revised_consumable_subcategories = {'cost_of_separately_managed_medical_supplies_dispensed':'cost_of_consumables_dispensed', 'cost_of_excess_separately_managed_medical_supplies_stocked': 'cost_of_excess_consumables_stocked', 'supply_chain':'supply_chain'}
+input_costs_new = input_costs.copy()
+input_costs_new['cost_subcategory'] = input_costs_new['cost_subcategory'].map(revised_consumable_subcategories).fillna(input_costs_new['cost_subcategory'])
+
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_new, _cost_category = 'medical consumables', _disaggregate_by_subgroup = False,
+ _year = list_of_years_for_plot,
+ _outputfilepath = figurespath, _scenario_dict = cost_scenarios)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs, _cost_category = 'human resources for health', _disaggregate_by_subgroup = False,
+ _year = list_of_years_for_plot,
+ _outputfilepath = figurespath, _scenario_dict = cost_scenarios)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs, _cost_category = 'medical equipment', _disaggregate_by_subgroup = False,
+ _year = list_of_years_for_plot,
+ _outputfilepath = figurespath, _scenario_dict = cost_scenarios)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs, _cost_category = 'facility operating cost', _disaggregate_by_subgroup = False,
+ _year = list_of_years_for_plot,
+ _outputfilepath = figurespath, _scenario_dict = cost_scenarios)
+
+
+# Figure 2: Estimated costs by year
+do_line_plot_of_cost(_df = input_costs_undiscounted, _cost_category='all',
+ _year=list_of_years_for_plot, _draws= [0],
+ disaggregate_by= 'cost_category',
+ _outputfilepath = figurespath)
+do_line_plot_of_cost(_df = input_costs_undiscounted, _cost_category='all',
+ _year=list_of_years_for_plot, _draws= [3],
+ disaggregate_by= 'cost_category',
+ _outputfilepath = figurespath)
+do_line_plot_of_cost(_df = input_costs_undiscounted, _cost_category='all',
+ _year=list_of_years_for_plot, _draws= [5],
+ disaggregate_by= 'cost_category',
+ _outputfilepath = figurespath)
+do_line_plot_of_cost(_df = input_costs_undiscounted, _cost_category='all',
+ _year=list_of_years_for_plot, _draws= [8],
+ disaggregate_by= 'cost_category',
+ _outputfilepath = figurespath)
+
+# Figure 3: Comparison of model-based cost estimates with actual expenditure recorded for 2018/19 and budget planned for 2020/21-2022/23
+
+# Figure 4: Total cost by scenario assuming 0% discount rate
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_undiscounted,
+ _cost_category = 'all',
+ _year=list_of_years_for_plot,
+ _disaggregate_by_subgroup = False,
+ _outputfilepath = figurespath,
+ _scenario_dict = cost_scenarios,
+ _add_figname_suffix = '_UNDISCOUNTED')
+
+# Figure 5: Total cost by scenario applying changing discount rates
+# Figure 4: Total cost by scenario assuming 0% discount rate
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs_variable_discounting,
+ _cost_category = 'all',
+ _year=list_of_years_for_plot,
+ _disaggregate_by_subgroup = False,
+ _outputfilepath = figurespath,
+ _scenario_dict = cost_scenarios,
+ _add_figname_suffix = '_VARIABLE_DISCOUNTING')
+
+
+cost_categories = ['human resources for health', 'medical consumables',
+ 'medical equipment', 'facility operating cost']
+draws = input_costs.draw.unique().tolist()
+colourmap_for_consumables = {'First-line ART regimen: adult':'#1f77b4',
+ 'Test, HIV EIA Elisa': '#ff7f0e',
+ 'VL Test': '#2ca02c',
+ 'Depot-Medroxyprogesterone Acetate 150 mg - 3 monthly': '#d62728',
+ 'Oxygen, 1000 liters, primarily with oxygen cylinders': '#9467bd',
+ 'Phenobarbital, 100 mg': '#8c564b',
+ 'Rotavirus vaccine': '#e377c2',
+ 'Carbamazepine 200mg_1000_CMST': '#7f7f7f',
+ 'Infant resuscitator, clear plastic + mask + bag_each_CMST': '#bcbd22',
+ 'Dietary supplements (country-specific)': '#17becf',
+ 'Tenofovir (TDF)/Emtricitabine (FTC), tablet, 300/200 mg': '#2b8cbe',
+ 'Pneumococcal vaccine': '#fdae61',
+ 'Pentavalent vaccine (DPT, Hep B, Hib)': '#d73027',
+ 'male circumcision kit, consumables (10 procedures)_1_IDA': '#756bb1',
+ 'Jadelle (implant), box of 2_CMST': '#ffdd44',
+ 'Urine analysis': '#66c2a5'}
+
+for _cat in cost_categories:
+ for _d in draws:
+ if _cat == 'medical consumables':
+ create_summary_treemap_by_cost_subgroup(_df = input_costs, _year = list_of_years_for_plot,
+ _cost_category = _cat, _draw = _d, _color_map=colourmap_for_consumables,
+ _label_fontsize= 8, _outputfilepath=figurespath)
+ else:
+ create_summary_treemap_by_cost_subgroup(_df=input_costs, _year=list_of_years_for_plot,
+ _cost_category=_cat, _draw=_d, _label_fontsize= 8.5,
+ _outputfilepath=figurespath)
+
+
+# Get tables for overview paper
+# -----------------------------------------------------------------------------------------------------------------------
+# Group data and aggregate cost for each draw and stat
+def generate_detail_cost_table(_groupby_var, _groupby_var_name, _longtable = False):
+ edited_input_costs = input_costs.copy()
+ edited_input_costs[_groupby_var] = edited_input_costs[_groupby_var].replace('_', ' ', regex=True)
+ edited_input_costs[_groupby_var] = edited_input_costs[_groupby_var].replace('%', '\%', regex=True)
+ edited_input_costs[_groupby_var] = edited_input_costs[_groupby_var].replace('&', '\&', regex=True)
+
+ grouped_costs = edited_input_costs.groupby(['cost_category', _groupby_var, 'draw', 'stat'])['cost'].sum()
+ # Format the 'cost' values before creating the LaTeX table
+ grouped_costs = grouped_costs.apply(lambda x: f"{float(x):,.0f}")
+ # Remove underscores from all column values
+
+ # Create a pivot table to restructure the data for LaTeX output
+ pivot_data = {}
+ for draw in [0, 3, 5, 8]:
+ draw_data = grouped_costs.xs(draw, level='draw').unstack(fill_value=0) # Unstack to get 'stat' as columns
+ # Concatenate 'mean' with 'lower-upper' in the required format
+ pivot_data[draw] = draw_data['mean'].astype(str) + ' [' + \
+ draw_data['lower'].astype(str) + '-' + \
+ draw_data['upper'].astype(str) + ']'
+
+ # Combine draw data into a single DataFrame
+ table_data = pd.concat([pivot_data[0], pivot_data[3], pivot_data[5], pivot_data[8]], axis=1, keys=['draw=0', 'draw=3', 'draw=5', 'draw=8']).reset_index()
+
+ # Rename columns for clarity
+ table_data.columns = ['Cost Category', _groupby_var_name, 'Actual', 'Expanded HRH', 'Improved consumable availability', 'Expanded HRH +\n Improved consumable availability']
+
+ # Replace '\n' with '\\' for LaTeX line breaks
+ #table_data['Real World'] = table_data['Real World'].apply(lambda x: x.replace("\n", "\\\\"))
+ #table_data['Perfect Health System'] = table_data['Perfect Health System'].apply(lambda x: x.replace("\n", "\\\\"))
+
+ # Convert to LaTeX format with horizontal lines after every row
+ latex_table = table_data.to_latex(
+ longtable=_longtable, # Use the longtable environment for large tables
+ column_format='|R{3cm}|R{3cm}|R{2.2cm}|R{2.2cm}|R{2.2cm}|R{2.2cm}|',
+ caption=f"Summarized Costs by Category and {_groupby_var_name}",
+ label=f"tab:cost_by_{_groupby_var}",
+ position="h",
+ index=False,
+ escape=False, # Prevent escaping special characters like \n
+ header=True
+ )
+
+ # Add \hline after the header and after every row for horizontal lines
+ latex_table = latex_table.replace("\\\\", "\\\\ \\hline") # Add \hline after each row
+ #latex_table = latex_table.replace("_", " ") # Add \hline after each row
+
+ # Specify the file path to save
+ latex_file_path = figurespath / f'cost_by_{_groupby_var}.tex'
+
+ # Write to a file
+ with open(latex_file_path, 'w') as latex_file:
+ latex_file.write(latex_table)
+
+ # Print latex for reference
+ print(latex_table)
+
+# Table : Cost by cost subcategory
+generate_detail_cost_table(_groupby_var = 'cost_subcategory', _groupby_var_name = 'Cost Subcategory')
+# Table : Cost by cost subgroup
+generate_detail_cost_table(_groupby_var = 'cost_subgroup', _groupby_var_name = 'Category Subgroup', _longtable = True)
+
+# Generate consumable inflow to outflow ratio figure
+# -----------------------------------------------------------------------------------------------------------------------
+# Estimate the stock to dispensed ratio from OpenLMIS data
+lmis_consumable_usage = pd.read_csv(path_for_consumable_resourcefiles / "ResourceFile_Consumables_availability_and_usage.csv")
+# Collapse individual facilities
+lmis_consumable_usage_by_item_level_month = lmis_consumable_usage.groupby(['category', 'item_code', 'district', 'fac_type_tlo', 'month'])[['closing_bal', 'dispensed', 'received']].sum()
+df = lmis_consumable_usage_by_item_level_month # Drop rows where monthly OpenLMIS data wasn't available
+df = df.loc[df.index.get_level_values('month') != "Aggregate"]
+opening_bal_january = df.loc[df.index.get_level_values('month') == 'January', 'closing_bal'] + \
+ df.loc[df.index.get_level_values('month') == 'January', 'dispensed'] - \
+ df.loc[df.index.get_level_values('month') == 'January', 'received']
+closing_bal_december = df.loc[df.index.get_level_values('month') == 'December', 'closing_bal']
+total_consumables_inflow_during_the_year = df.loc[df.index.get_level_values('month') != 'January', 'received'].groupby(level=[0,1,2,3]).sum() +\
+ opening_bal_january.reset_index(level='month', drop=True) -\
+ closing_bal_december.reset_index(level='month', drop=True)
+total_consumables_outflow_during_the_year = df['dispensed'].groupby(level=[0,1,2,3]).sum()
+inflow_to_outflow_ratio = total_consumables_inflow_during_the_year.div(total_consumables_outflow_during_the_year, fill_value=1)
+
+# Edit outlier ratios
+inflow_to_outflow_ratio.loc[inflow_to_outflow_ratio < 1] = 1 # Ratio can't be less than 1
+inflow_to_outflow_ratio.loc[inflow_to_outflow_ratio > inflow_to_outflow_ratio.quantile(0.95)] = inflow_to_outflow_ratio.quantile(0.95) # Trim values greater than the 95th percentile
+#average_inflow_to_outflow_ratio_ratio = inflow_to_outflow_ratio.mean()
+inflow_to_outflow_ratio = inflow_to_outflow_ratio.reset_index().rename(columns = {0:'inflow_to_outflow_ratio'})
+
+def plot_inflow_to_outflow_ratio(_df, groupby_var, _outputfilepath):
+ # Plot the bar plot
+ plt.figure(figsize=(10, 6))
+ sns.barplot(data=_df , x=groupby_var, y= 'inflow_to_outflow_ratio', errorbar=None)
+
+ # Add points representing the distribution of individual values
+ sns.stripplot(data=_df, x=groupby_var, y='inflow_to_outflow_ratio', color='black', size=5, alpha=0.2)
+
+ # Set labels and title
+ plt.xlabel(groupby_var)
+ plt.ylabel('Inflow to Outflow Ratio')
+ plt.title('Average Inflow to Outflow Ratio by ' + f'{groupby_var}')
+ plt.xticks(rotation=45)
+
+ # Show plot
+ plt.tight_layout()
+ plt.savefig(_outputfilepath / 'inflow_to_outflow_ratio_by' f'{groupby_var}' )
+
+plot_inflow_to_outflow_ratio(inflow_to_outflow_ratio, 'fac_type_tlo', _outputfilepath = figurespath)
+plot_inflow_to_outflow_ratio(inflow_to_outflow_ratio, 'district', _outputfilepath = figurespath)
+plot_inflow_to_outflow_ratio(inflow_to_outflow_ratio, 'item_code', _outputfilepath = figurespath)
+plot_inflow_to_outflow_ratio(inflow_to_outflow_ratio, 'category', _outputfilepath = figurespath)
+
+print(f"Inflow to Outflow ratio by consumable varies from "
+ f"{round(min(inflow_to_outflow_ratio.groupby('item_code')['inflow_to_outflow_ratio'].mean()),2)} "
+ f"to {round(max(inflow_to_outflow_ratio.groupby('item_code')['inflow_to_outflow_ratio'].mean()),2)}")
+
+inflow_to_outflow_ratio_by_item = inflow_to_outflow_ratio.groupby('item_code')['inflow_to_outflow_ratio'].mean().reset_index().rename(columns = {0: 'inflow_to_outflow_ratio'})
+inflow_to_outflow_ratio_by_item[inflow_to_outflow_ratio_by_item.inflow_to_outflow_ratio == min(inflow_to_outflow_ratio_by_item.inflow_to_outflow_ratio)]['item_code']
+inflow_to_outflow_ratio_by_item[inflow_to_outflow_ratio_by_item.inflow_to_outflow_ratio == max(inflow_to_outflow_ratio_by_item.inflow_to_outflow_ratio)]['item_code']
+
diff --git a/src/scripts/costing/costing_validation.py b/src/scripts/costing/costing_validation.py
new file mode 100644
index 0000000000..4b4432a9b7
--- /dev/null
+++ b/src/scripts/costing/costing_validation.py
@@ -0,0 +1,570 @@
+import argparse
+from pathlib import Path
+from tlo import Date
+from collections import Counter, defaultdict
+
+import calendar
+import datetime
+import os
+import textwrap
+
+import matplotlib.pyplot as plt
+from matplotlib.ticker import FuncFormatter
+import numpy as np
+import pandas as pd
+import ast
+import math
+
+from tlo.analysis.utils import (
+ extract_params,
+ extract_results,
+ get_scenario_info,
+ get_scenario_outputs,
+ load_pickled_dataframes,
+ make_age_grp_lookup,
+ make_age_grp_types,
+ summarize,
+ create_pickles_locally,
+ parse_log_file,
+ unflatten_flattened_multi_index_in_logging
+)
+from tlo.methods.healthsystem import get_item_code_from_item_name
+from scripts.costing.cost_estimation import (estimate_input_cost_of_scenarios,
+ do_stacked_bar_plot_of_cost_by_category)
+
+# Define a timestamp for script outputs
+timestamp = datetime.datetime.now().strftime("_%Y_%m_%d_%H_%M")
+
+# Print the start time of the script
+print('Script Start', datetime.datetime.now().strftime('%H:%M'))
+
+# Establish common paths
+resourcefilepath = Path("./resources")
+
+# Steps: 1. Create a mapping of data labels in model_costing and relevant calibration data, 2. Create a dataframe with model_costs and calibration costs;
+# Load costing resourcefile
+workbook_cost = pd.read_excel((resourcefilepath / "costing/ResourceFile_Costing.xlsx"),
+ sheet_name=None)
+# Prepare data for calibration
+calibration_data = workbook_cost["resource_mapping_r7_summary"]
+# Make sure values are numeric
+budget_columns = ['BUDGETS (USD) (Jul 2019 - Jun 2020)', 'BUDGETS (USD) (Jul 2020 - Jun 2021)',
+ 'BUDGETS (USD) (Jul 2021 - Jun 2022)']
+expenditure_columns = ['EXPENDITURE (USD) (Jul 2018 - Jun 2019)']
+calibration_data[budget_columns + expenditure_columns] = calibration_data[budget_columns + expenditure_columns].apply(lambda x: pd.to_numeric(x, errors='coerce'))
+# For calibration to budget figures, we take the maximum value across the three years in the RM to provide an
+# the maximum of the budget between 2020 and 2022 provides the upper limit to calibrate to (expenditure providing the lower limit)
+calibration_data['max_annual_budget_2020-22'] = calibration_data[budget_columns].max(axis=1, skipna = True)
+calibration_data = calibration_data.rename(columns = {'EXPENDITURE (USD) (Jul 2018 - Jun 2019)': 'actual_expenditure_2019',
+ 'Calibration_category': 'calibration_category'})
+calibration_data = calibration_data[['calibration_category','actual_expenditure_2019', 'max_annual_budget_2020-22']]
+calibration_data = calibration_data.groupby('calibration_category')[['actual_expenditure_2019', 'max_annual_budget_2020-22']].sum().reset_index()
+# Repeat this dataframe three times to map to the lower, upper and mean stats in the cost data
+calibration_data1 = calibration_data.copy()
+calibration_data1['stat'] = 'lower'
+calibration_data2 = calibration_data.copy()
+calibration_data2['stat'] = 'mean'
+calibration_data3 = calibration_data.copy()
+calibration_data3['stat'] = 'upper'
+calibration_data = pd.concat([calibration_data1, calibration_data2, calibration_data3], axis = 0)
+calibration_data = calibration_data.set_index(['calibration_category', 'stat'])
+
+# %%
+# Estimate cost for validation
+#-----------------------------
+# Load result files
+resourcefilepath = Path("./resources")
+outputfilepath = Path('./outputs/t.mangal@imperial.ac.uk')
+#results_folder = get_scenario_outputs('hss_elements-2024-11-12T172311Z.py', outputfilepath)[0] # November 2024 runs
+results_folder = get_scenario_outputs('htm_and_hss_runs-2025-01-16T135243Z.py', outputfilepath)[0] # January 2025 runs
+
+# Estimate costs for 2018
+input_costs = estimate_input_cost_of_scenarios(results_folder, resourcefilepath, _years = [2018], _draws = [0], summarize = True, cost_only_used_staff=False)
+#input_costs = input_costs[input_costs.year == 2018]
+
+# Manually create a dataframe of model costs and relevant calibration values
+def assign_item_codes_to_consumables(_df):
+ path_for_consumable_resourcefiles = resourcefilepath / "healthsystem/consumables"
+ # Retain only consumable costs
+ _df = _df[_df['cost_category'] == 'medical consumables']
+
+ '''
+ consumables_dict = pd.read_csv(path_for_consumable_resourcefiles / 'ResourceFile_consumables_matched.csv', low_memory=False,
+ encoding="ISO-8859-1")[['item_code', 'consumable_name_tlo']]
+ consumables_dict = consumables_dict.rename(columns = {'item_code': 'Item_Code'})
+ consumables_dict = dict(zip(consumables_dict['consumable_name_tlo'], consumables_dict['Item_Code']))
+ '''
+
+ # Create dictionary mapping item_codes to consumables names
+ consumables_df = workbook_cost["consumables"]
+ consumables_df = consumables_df.rename(columns=consumables_df.iloc[0])
+ consumables_df = consumables_df[['Item_Code', 'Consumable_name_tlo']].reset_index(
+ drop=True).iloc[1:]
+ consumables_df = consumables_df[consumables_df['Item_Code'].notna()]
+ consumables_dict = dict(zip(consumables_df['Consumable_name_tlo'], consumables_df['Item_Code']))
+
+ # Replace consumable_name_tlo with item_code
+ _df = _df.copy()
+ _df['cost_subgroup'] = _df['cost_subgroup'].map(consumables_dict)
+
+ return _df
+
+def get_calibration_relevant_subset_of_costs(_df, _col, _col_value, _calibration_category):
+ if (len(_col_value) == 1):
+ _df = _df[_df[_col] == _col_value[0]]
+ else:
+ _df = _df[_df[_col].isin(_col_value)]
+ _df['calibration_category'] = _calibration_category
+ return _df.groupby(['calibration_category' ,'stat'])['cost'].sum()
+
+'''
+def get_calibration_relevant_subset_of_consumables_cost(_df, item):
+ for col in ['Item_Code', 'Final_price_per_chosen_unit (USD, 2023)', 'excess_stock_proportion_of_dispensed','item_code']:
+ try:
+ _df = _df.drop(columns = col)
+ except:
+ pass
+ _df.columns = pd.MultiIndex.from_tuples(_df.columns)
+ _df = _df.melt(id_vars = ['year', 'Item_Code'], var_name=['draw', 'stat'], value_name='value')
+ _df = _df[_df['Item_Code'].isin(item)]
+ _df = _df.groupby(['year', 'draw', 'stat'])['value'].sum()
+ return _df.reset_index()
+def merged_calibration_relevant_consumables_costs(item, category):
+ merged_df = pd.merge(get_calibration_relevant_subset_of_consumables_cost(cost_of_consumables_dispensed, item),
+ get_calibration_relevant_subset_of_consumables_cost(cost_of_excess_consumables_stocked, item),
+ on=['year', 'draw', 'stat'], how='outer', suffixes=('_dispensed', '_excess_stock'))
+ # Fill any missing values in the value columns with 0 (for cases where only one dataframe has a value)
+ # and sum to get total consumable cost
+ merged_df['value'] = merged_df['value_dispensed'].fillna(0) + merged_df['value_excess_stock'].fillna(0)
+ merged_df['calibration_category'] = category
+ return merged_df.set_index(['calibration_category', 'stat'])['value']
+
+def first_positive(series):
+ return next((x for x in series if pd.notna(x) and x > 0), np.nan)
+
+def get_calibration_relevant_subset_of_other_costs(_df, _subcategory, _calibration_category):
+ new_data = get_calibration_relevant_subset(_df[_df['Cost_Sub-category'].isin(_subcategory)]).groupby('stat')['value'].sum()
+ new_data = new_data.reset_index()
+ new_data['calibration_category'] = _calibration_category
+ new_data = new_data.rename(columns = {'value':'model_cost'})
+ return new_data.set_index(['calibration_category', 'stat'])['model_cost']
+'''
+
+# Consumables
+#-----------------------------------------------------------------------------------------------------------------------
+calibration_data['model_cost'] = np.nan
+consumables_costs_by_item_code = assign_item_codes_to_consumables(input_costs)
+consumable_list = pd.read_csv(resourcefilepath / 'healthsystem' / 'consumables' / 'ResourceFile_Consumables_Items_and_Packages.csv')
+def get_item_code(item):
+ return get_item_code_from_item_name(consumable_list, item)
+
+# Malaria consumables
+irs = [get_item_code('Indoor residual spraying drugs/supplies to service a client')]
+bednets = [get_item_code('Insecticide-treated net')]
+antimalarials = [get_item_code('Lumefantrine 120mg/Artemether 20mg, 30x18_540_CMST'),
+ get_item_code('Injectable artesunate'),
+ get_item_code('Fansidar (sulphadoxine / pyrimethamine tab)')]
+malaria_rdts = [get_item_code('Malaria test kit (RDT)')]
+
+# HIV consumables
+hiv_screening = [get_item_code('Test, HIV EIA Elisa'), get_item_code('VL Test'), get_item_code('CD4 test')]
+
+art = [get_item_code("First-line ART regimen: adult"), get_item_code("Cotrimoxizole, 960mg pppy"), # adult
+ get_item_code("First line ART regimen: older child"), get_item_code("Cotrimoxazole 120mg_1000_CMST"), # Older children
+ get_item_code("First line ART regimen: young child"), # younger children (also get cotrimoxazole 120mg
+ get_item_code('Sulfamethoxazole + trimethropin, tablet 400 mg + 80 mg'),
+ get_item_code("Tenofovir (TDF)/Emtricitabine (FTC), tablet, 300/200 mg"), # Adult prep
+ get_item_code("Nevirapine, oral solution, 10 mg/ml")] # infant prep
+
+circumcision = [get_item_code('male circumcision kit, consumables (10 procedures)_1_IDA')]
+
+# Tuberculosis consumables
+tb_tests = [get_item_code("ZN Stain"), get_item_code("Sputum container"), get_item_code("Microscope slides, lime-soda-glass, pack of 50"),
+ get_item_code("Xpert"), get_item_code("Lead rubber x-ray protective aprons up to 150kVp 0.50mm_each_CMST"),
+ get_item_code("X-ray"), get_item_code("MGIT960 Culture and DST"),
+ get_item_code("Solid culture and DST")]
+# consider removing X-ray
+tb_treatment = [get_item_code("Cat. I & III Patient Kit A"), # adult primary
+ get_item_code("Cat. I & III Patient Kit B"), # child primary
+ get_item_code("Cat. II Patient Kit A1"), # adult secondary
+ get_item_code("Cat. II Patient Kit A2"), # child secondary
+ get_item_code("Treatment: second-line drugs"), # MDR
+ get_item_code("Isoniazid/Pyridoxine, tablet 300 mg"), # IPT
+ get_item_code("Isoniazid/Rifapentine")] # 3 HP
+# Family planning consumables
+other_family_planning = [get_item_code("Levonorgestrel 0.15 mg + Ethinyl estradiol 30 mcg (Microgynon), cycle"), # pill
+ get_item_code("IUD, Copper T-380A"), # IUD
+ get_item_code("Depot-Medroxyprogesterone Acetate 150 mg - 3 monthly"), # injection
+ get_item_code("Jadelle (implant), box of 2_CMST"), # implant
+ get_item_code('Implanon (Etonogestrel 68 mg)'), # implant - not currently in use in the model
+ get_item_code("Atropine sulphate 600 micrograms/ml, 1ml_each_CMST")] # female sterilization
+condoms = [get_item_code("Condom, male"),
+ get_item_code("Female Condom_Each_CMST")]
+# Undernutrition
+undernutrition = [get_item_code('Supplementary spread, sachet 92g/CAR-150'),
+ get_item_code('Complementary feeding--education only drugs/supplies to service a client'),
+ get_item_code('SAM theraputic foods'),
+ get_item_code('SAM medicines'),
+ get_item_code('Therapeutic spread, sachet 92g/CAR-150'),
+ get_item_code('F-100 therapeutic diet, sach., 114g/CAR-90')]
+# Cervical cancer
+cervical_cancer = [get_item_code('Specimen container'),
+ get_item_code('Biopsy needle'),
+ get_item_code('Cyclophosphamide, 1 g')]
+# Vaccines
+vaccines = [get_item_code("Syringe, autodisposable, BCG, 0.1 ml, with needle"),
+ get_item_code("Polio vaccine"),
+ get_item_code("Pentavalent vaccine (DPT, Hep B, Hib)"),
+ get_item_code("Rotavirus vaccine"),
+ get_item_code("Measles vaccine"),
+ get_item_code("Pneumococcal vaccine"),
+ get_item_code("HPV vaccine"),
+ get_item_code("Tetanus toxoid, injection")] # not sure if this should be included
+
+other_drugs = set(consumables_costs_by_item_code['cost_subgroup'].unique()) - set(irs) - set(bednets) - set(undernutrition) - set(other_family_planning) - set(vaccines) \
+ - set(art) - set(tb_treatment) - set(antimalarials) - set(malaria_rdts) - set(hiv_screening)\
+ - set(condoms) - set(tb_tests) # - set(cervical_cancer)
+
+# Note that the main ARV regimen in 2018 was tenofovir/lamivudine/efavirenz as opposed to Tenofovir/Lamivudine/Dolutegravir as used in the RF_Costing. The price of this
+# was $82 per year (80/(0.103*365)) times what's estimated by the model so let's update this
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = art, _calibration_category = 'Antiretrovirals')* 82/(0.103*365))
+# Other consumables costs do not need to be adjusted
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = irs, _calibration_category = 'Indoor Residual Spray'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = bednets, _calibration_category = 'Bednets'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = undernutrition, _calibration_category = 'Undernutrition commodities'))
+#calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = cervical_cancer, _calibration_category = 'Cervical Cancer'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = other_family_planning, _calibration_category = 'Other family planning commodities'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = vaccines, _calibration_category = 'Vaccines'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = tb_treatment, _calibration_category = 'TB Treatment'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = antimalarials, _calibration_category = 'Antimalarials'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = malaria_rdts, _calibration_category = 'Malaria RDTs'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = hiv_screening, _calibration_category = 'HIV Screening/Diagnostic Tests'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = condoms, _calibration_category = 'Condoms and Lubricants'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = tb_tests, _calibration_category = 'TB Tests (including RDTs)'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = other_drugs, _calibration_category = 'Other Drugs, medical supplies, and commodities'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = consumables_costs_by_item_code, _col = 'cost_subgroup', _col_value = circumcision, _calibration_category = 'Voluntary Male Medical Circumcision'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = input_costs, _col = 'cost_subcategory', _col_value = ['supply_chain'], _calibration_category = 'Supply Chain'))
+
+# HR
+#-----------------------------------------------------------------------------------------------------------------------
+hr_costs = input_costs[input_costs['cost_category'] == 'human resources for health']
+#ratio_of_all_to_used_staff = total_salary_for_all_staff[(0,2018)]/total_salary_for_staff_used_in_scenario[( 0, 'lower')][2018]
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = hr_costs, _col = 'cost_subcategory', _col_value = ['salary_for_all_staff'], _calibration_category = 'Health Worker Salaries'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = hr_costs, _col = 'cost_subcategory', _col_value = ['preservice_training_and_recruitment_cost_for_attrited_workers'], _calibration_category = 'Health Worker Training - Pre-Service')) # TODO remove recruitment costs?
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = hr_costs, _col = 'cost_subcategory', _col_value = ['inservice_training_cost_for_all_staff'], _calibration_category = 'Health Worker Training - In-Service'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = hr_costs, _col = 'cost_subcategory', _col_value = ['mentorship_and_supportive_cost_for_all_staff'], _calibration_category = 'Mentorships & Supportive Supervision'))
+
+# Equipment
+equipment_costs = input_costs[input_costs['cost_category'] == 'medical equipment']
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = equipment_costs, _col = 'cost_subcategory', _col_value = ['replacement_cost_annual_total'], _calibration_category = 'Medical Equipment - Purchase'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = equipment_costs, _col = 'cost_subcategory',
+ _col_value = ['service_fee_annual_total', 'spare_parts_annual_total','major_corrective_maintenance_cost_annual_total'],
+ _calibration_category = 'Medical Equipment - Maintenance'))
+#calibration_data[calibration_data['calibration_category'] == 'Vehicles - Purchase and Maintenance'] = get_calibration_relevant_subset()
+#calibration_data[calibration_data['calibration_category'] == 'Vehicles - Purchase and Maintenance'] = get_calibration_relevant_subset()
+
+# Facility operation costs
+#-----------------------------------------------------------------------------------------------------------------------
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = input_costs, _col = 'cost_subgroup', _col_value = ['Electricity', 'Water', 'Cleaning', 'Security', 'Food for inpatient cases', 'Facility management'], _calibration_category = 'Facility utility bills'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = input_costs, _col = 'cost_subgroup', _col_value = ['Building maintenance'], _calibration_category = 'Infrastructure - Rehabilitation'))
+calibration_data['model_cost'] = calibration_data['model_cost'].fillna(get_calibration_relevant_subset_of_costs(_df = input_costs, _col = 'cost_subgroup', _col_value = ['Vehicle maintenance', 'Ambulance fuel'], _calibration_category = 'Vehicles - Fuel and Maintenance'))
+
+# %%
+# 3. Create calibration plot
+list_of_consumables_costs_for_calibration_only_hiv = ['Voluntary Male Medical Circumcision', 'HIV Screening/Diagnostic Tests', 'Antiretrovirals']
+list_of_consumables_costs_for_calibration_without_hiv =['Indoor Residual Spray', 'Bednets', 'Malaria RDTs', 'Antimalarials', 'TB Tests (including RDTs)', 'TB Treatment', 'Vaccines',
+ 'Condoms and Lubricants', 'Other family planning commodities',
+ 'Undernutrition commodities', 'Other Drugs, medical supplies, and commodities']
+list_of_hr_costs_for_calibration = ['Health Worker Salaries', 'Health Worker Training - In-Service', 'Health Worker Training - Pre-Service', 'Mentorships & Supportive Supervision']
+list_of_equipment_costs_for_calibration = ['Medical Equipment - Purchase', 'Medical Equipment - Maintenance']
+list_of_operating_costs_for_calibration = ['Facility utility bills', 'Infrastructure - Rehabilitation', 'Vehicles - Maintenance','Vehicles - Fuel and Maintenance']
+
+# Create folders to store results
+costing_outputs_folder = Path('./outputs/costing')
+if not os.path.exists(costing_outputs_folder):
+ os.makedirs(costing_outputs_folder)
+figurespath = costing_outputs_folder / "figures_post_jan2025fix"
+if not os.path.exists(figurespath):
+ os.makedirs(figurespath)
+calibration_outputs_folder = Path(figurespath / 'calibration')
+if not os.path.exists(calibration_outputs_folder):
+ os.makedirs(calibration_outputs_folder)
+
+def do_cost_calibration_plot(_df, _costs_included, _xtick_fontsize = 10):
+ # Filter the dataframe
+ _df = _df[(_df.model_cost.notna()) & (_df.index.get_level_values(0).isin(_costs_included))]
+
+ # Reorder the first level of the index based on _costs_included while keeping the second level intact
+ _df.index = pd.MultiIndex.from_arrays([
+ pd.CategoricalIndex(_df.index.get_level_values(0), categories=_costs_included, ordered=True),
+ _df.index.get_level_values(1)
+ ])
+ _df = _df.sort_index() # Apply the custom order by sorting the DataFrame
+
+ # For df_mean
+ df_mean = _df.loc[_df.index.get_level_values('stat') == 'mean'].reset_index(level='stat', drop=True)/1e6
+ total_mean = pd.DataFrame(df_mean.sum()).T # Calculate the total and convert it to a DataFrame
+ total_mean.index = ['Total'] # Name the index of the total row as 'Total'
+ df_mean = pd.concat([df_mean, total_mean], axis=0) # Concatenate the total row
+
+ # For df_lower
+ df_lower = _df.loc[_df.index.get_level_values('stat') == 'lower'].reset_index(level='stat', drop=True)/1e6
+ total_lower = pd.DataFrame(df_lower.sum()).T # Calculate the total and convert it to a DataFrame
+ total_lower.index = ['Total'] # Name the index of the total row as 'Total'
+ df_lower = pd.concat([df_lower, total_lower], axis=0) # Concatenate the total row
+
+ # For df_upper
+ df_upper = _df.loc[_df.index.get_level_values('stat') == 'upper'].reset_index(level='stat', drop=True)/1e6
+ total_upper = pd.DataFrame(df_upper.sum()).T # Calculate the total and convert it to a DataFrame
+ total_upper.index = ['Total'] # Name the index of the total row as 'Total'
+ df_upper = pd.concat([df_upper, total_upper], axis=0) # Concatenate the total row
+
+ # Create the dot plot
+ plt.figure(figsize=(12, 8))
+
+ # Plot model_cost as dots with confidence interval error bars
+ yerr_lower = (df_mean['model_cost'] - df_lower['model_cost']).clip(lower = 0)
+ yerr_upper = (df_upper['model_cost'] - df_mean['model_cost']).clip(lower = 0)
+ plt.errorbar(df_mean.index, df_mean['model_cost'],
+ yerr=[yerr_lower, yerr_upper],
+ fmt='o', label='Model Cost', ecolor='gray', capsize=5, color='saddlebrown')
+
+ # Plot annual_expenditure_2019 and max_annual_budget_2020-22 as dots
+ plt.plot(df_mean.index, df_mean['actual_expenditure_2019'], 'bo', label='Actual Expenditure 2019', markersize=8)
+ plt.plot(df_mean.index, df_mean['max_annual_budget_2020-22'], 'go', label='Max Annual Budget 2020-22', markersize=8)
+
+ # Draw a blue line between annual_expenditure_2019 and max_annual_budget_2020-22
+ plt.vlines(df_mean.index, df_mean['actual_expenditure_2019'], df_mean['max_annual_budget_2020-22'], color='blue',
+ label='Expenditure-Budget Range')
+
+ # Add labels to the model_cost dots (yellow color, slightly shifted right)
+ for i, (x, y) in enumerate(zip(df_mean.index, df_mean['model_cost'])):
+ plt.text(i + 0.05, y, f'{y:.2f}', ha='left', va='bottom', fontsize=9,
+ color='saddlebrown') # label model_cost values
+
+ # Add labels and title
+ cost_subcategory = [name for name in globals() if globals()[name] is _costs_included][0]
+ cost_subcategory = cost_subcategory.replace('list_of_', '').replace('_for_calibration', '')
+ plt.xlabel('Cost Sub-Category')
+ plt.ylabel('Costs (USD), millions')
+ plt.title(f'Model Cost vs Annual Expenditure 2019 and Max(Annual Budget 2020-22)\n {cost_subcategory}')
+
+ # Set a white background and black border
+ plt.grid(False)
+ ax = plt.gca() # Get current axes
+ ax.set_facecolor('white') # Set the background color to white
+ for spine in ax.spines.values(): # Iterate over all borders (spines)
+ spine.set_edgecolor('black') # Set the border color to black
+ spine.set_linewidth(1.5) # Adjust the border width if desired
+
+ # Customize x-axis labels for readability
+ max_label_length = 15 # Define a maximum label length for wrapping
+ wrapped_labels = [textwrap.fill(str(label), max_label_length) for label in df_mean.index]
+ plt.xticks(ticks=range(len(wrapped_labels)), labels=wrapped_labels, rotation=45, ha='right', fontsize=_xtick_fontsize)
+
+ # Adding a legend
+ plt.legend(loc='upper left', bbox_to_anchor=(1, 1), fontsize=10)
+
+ # Tight layout and save the figure
+ plt.tight_layout()
+ plt.savefig(calibration_outputs_folder / f'calibration_dot_plot_{cost_subcategory}.png', dpi=100,
+ bbox_inches='tight')
+ plt.close()
+
+# Call the function for each variable and cost list
+all_consumable_costs = list_of_consumables_costs_for_calibration_only_hiv + list_of_consumables_costs_for_calibration_without_hiv + ['Supply Chain']
+all_calibration_costs = all_consumable_costs + list_of_hr_costs_for_calibration + list_of_equipment_costs_for_calibration + list_of_operating_costs_for_calibration
+
+do_cost_calibration_plot(calibration_data,list_of_consumables_costs_for_calibration_without_hiv)
+do_cost_calibration_plot(calibration_data,list_of_consumables_costs_for_calibration_only_hiv)
+do_cost_calibration_plot(calibration_data,all_consumable_costs)
+do_cost_calibration_plot(calibration_data, list_of_hr_costs_for_calibration)
+do_cost_calibration_plot(calibration_data, list_of_equipment_costs_for_calibration)
+do_cost_calibration_plot(calibration_data, list_of_operating_costs_for_calibration)
+do_cost_calibration_plot(calibration_data,all_calibration_costs, _xtick_fontsize = 7)
+
+# Extract calibration data table for manuscript appendix
+calibration_data_extract = calibration_data[calibration_data.index.get_level_values(1) == 'mean']
+calibration_data_extract = calibration_data_extract.droplevel(level=1).reset_index()
+# Create a higher level cost category in the calibration data
+calibration_categories_dict = {'Other Drugs, medical supplies, and commodities': 'medical consumables',
+'Program Management & Administration': 'Not represented in TLO model',
+'Non-EHP consumables': 'Not represented in TLO model',
+'Voluntary Male Medical Circumcision': 'medical consumables',
+'Indoor Residual Spray': 'medical consumables',
+'Bednets': 'medical consumables',
+'Antimalarials': 'medical consumables',
+'Undernutrition commodities': 'medical consumables',
+'Cervical Cancer': 'medical consumables',
+'Condoms and Lubricants': 'medical consumables',
+'Other family planning commodities': 'medical consumables',
+'TB Tests (including RDTs)': 'medical consumables',
+'TB Treatment': 'medical consumables',
+'Vaccines': 'medical consumables',
+'Malaria RDTs': 'medical consumables',
+'HIV Screening/Diagnostic Tests': 'medical consumables',
+'Antiretrovirals': 'medical consumables',
+'Health Worker Salaries': 'human resources for health',
+'Health Worker Training - In-Service': 'human resources for health',
+'Health Worker Training - Pre-Service': 'human resources for health',
+'Mentorships & Supportive Supervision': 'human resources for health',
+'Facility utility bills': 'facility operating cost',
+'Infrastructure - New Builds': 'Not represented in TLO model',
+'Infrastructure - Rehabilitation': 'facility operating cost',
+'Infrastructure - Upgrades': 'Not represented in TLO model',
+'Medical Equipment - Maintenance': 'medical equipment',
+'Medical Equipment - Purchase': 'medical equipment',
+'Vehicles - Fuel and Maintenance': 'facility operating cost',
+'Vehicles - Purchase': 'Not represented in TLO model',
+'Vehicles - Fuel and Maintenance (Beyond Government and CHAM)': 'Not represented in TLO model',
+'Supply Chain': 'medical consumables',
+'Supply Chain - non-EHP consumables': 'Not represented in TLO model',
+'Unclassified': 'Not represented in TLO model'}
+calibration_data_extract['cost_category'] = calibration_data_extract['calibration_category'].map(calibration_categories_dict)
+
+calibration_data_extract['deviation_from_expenditure'] = abs(
+ (calibration_data_extract['model_cost'] - calibration_data_extract['actual_expenditure_2019'])
+ /calibration_data_extract['actual_expenditure_2019'])
+calibration_data_extract['deviation_from_budget'] = abs(
+ (calibration_data_extract['model_cost'] - calibration_data_extract['max_annual_budget_2020-22'])
+ /calibration_data_extract['max_annual_budget_2020-22'])
+calibration_data_extract['Absolute deviation of estimated cost from data (%)'] = (
+ calibration_data_extract[['deviation_from_expenditure', 'deviation_from_budget']]
+ .min(axis=1, skipna=True) # Use axis=1 to compute the minimum row-wise.
+)
+
+# Format the deviation as a percentage with 2 decimal points
+calibration_data_extract['Absolute deviation of estimated cost from data (%)'] = (
+ calibration_data_extract['Absolute deviation of estimated cost from data (%)']
+ .map(lambda x: f"{x * 100:.2f}%")
+)
+calibration_data_extract.loc[calibration_data_extract['Absolute deviation of estimated cost from data (%)'] == 'nan%', 'Absolute deviation of estimated cost from data (%)'] = 'NA'
+# Replace if calibration is fine
+calibration_condition_met = ((calibration_data_extract['model_cost'] > calibration_data_extract[['actual_expenditure_2019', 'max_annual_budget_2020-22']].min(axis=1)) &
+ (calibration_data_extract['model_cost'] < calibration_data_extract[['actual_expenditure_2019', 'max_annual_budget_2020-22']].max(axis=1)))
+
+calibration_data_extract.loc[calibration_condition_met,
+ 'Absolute deviation of estimated cost from data (%)'
+] = 'Within target range'
+
+calibration_data_extract.loc[calibration_data_extract['model_cost'].isna(), 'model_cost'] = 'NA'
+
+calibration_data_extract = calibration_data_extract.sort_values(by=['cost_category', 'calibration_category'])
+calibration_data_extract = calibration_data_extract[['cost_category', 'calibration_category', 'actual_expenditure_2019', 'max_annual_budget_2020-22', 'model_cost', 'Absolute deviation of estimated cost from data (%)']]
+calibration_data_extract = calibration_data_extract.rename(columns = {'cost_category': 'Cost Category',
+ 'calibration_category': 'Relevant RM group',
+ 'actual_expenditure_2019': 'Recorded Expenditure (FY 2018/19)',
+ 'max_annual_budget_2020-22': 'Maximum Recorded Annual Budget (FY 2019/20 - 2021/22)',
+ 'model_cost': 'Estimated cost (TLO Model, 2018)'
+ })
+
+calibration_data_extract.to_csv(figurespath / 'calibration/calibration.csv')
+def convert_df_to_latex(_df, _longtable = False, numeric_columns = []):
+ _df['Relevant RM group'] = _df['Relevant RM group'].str.replace('&', r'\&', regex=False)
+ # Format numbers to the XX,XX,XXX.XX format for all numeric columns
+ _df[numeric_columns] = _df[numeric_columns].applymap(lambda x: f"{x:,.2f}" if isinstance(x, (int, float)) else x)
+
+ # Convert to LaTeX format with horizontal lines after every row
+ latex_table = _df.to_latex(
+ longtable=_longtable, # Use the longtable environment for large tables
+ column_format='|R{3.5cm}|R{3.5cm}|R{2.1cm}|R{2.1cm}|R{2.1cm}|R{2.1cm}|',
+ caption=f"Comparison of Model Estimates with Resource Mapping data",
+ label=f"tab:calibration_breakdown",
+ position="h",
+ index=False,
+ escape=False, # Prevent escaping special characters like \n
+ header=True
+ )
+
+ # Add \hline after the header and after every row for horizontal lines
+ latex_table = latex_table.replace("\\\\", "\\\\ \\hline") # Add \hline after each row
+ latex_table = latex_table.replace("%", "\%") # Add \hline after each row
+ latex_table = latex_table.replace("Program Management & Administration", "Program Management \& Administration") # Add \hline after each row
+ latex_table = latex_table.replace("Mentorships & Supportive Supervision", "Mentorships \& Supportive Supervision") # Add \hline after each row
+
+ # latex_table = latex_table.replace("_", " ") # Add \hline after each row
+
+ # Specify the file path to save
+ latex_file_path = calibration_outputs_folder / f'calibration_breakdown.tex'
+
+ # Write to a file
+ with open(latex_file_path, 'w') as latex_file:
+ latex_file.write(latex_table)
+
+ # Print latex for reference
+ print(latex_table)
+
+convert_df_to_latex(calibration_data_extract, _longtable = True, numeric_columns = ['Recorded Expenditure (FY 2018/19)',
+ 'Maximum Recorded Annual Budget (FY 2019/20 - 2021/22)',
+ 'Estimated cost (TLO Model, 2018)'])
+
+# Stacked bar charts to represent all cost sub-groups
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs, _cost_category = 'medical consumables',
+ _disaggregate_by_subgroup = True,
+ _outputfilepath = calibration_outputs_folder)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs, _cost_category = 'human resources for health',
+ _disaggregate_by_subgroup = True,
+ _outputfilepath = calibration_outputs_folder)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs, _cost_category = 'medical equipment',
+ _disaggregate_by_subgroup = True,
+ _outputfilepath = calibration_outputs_folder)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs, _cost_category = 'other',
+ _disaggregate_by_subgroup = True,
+ _outputfilepath = calibration_outputs_folder)
+do_stacked_bar_plot_of_cost_by_category(_df = input_costs, _cost_category = 'all',
+ _disaggregate_by_subgroup = False,
+ _outputfilepath = calibration_outputs_folder)
+
+
+
+'''
+
+# Calibration scatter plots
+def do_cost_calibration_plot(_df, _costs_included, _calibration_var):
+ _df = _df[(_df.model_cost.notna()) & (_df.index.get_level_values(0).isin(_costs_included))]
+ df_mean = _df.loc[_df.index.get_level_values('stat') == 'mean'].reset_index(level='stat', drop=True)
+ df_lower = _df.loc[_df.index.get_level_values('stat') == 'lower'].reset_index(level='stat', drop=True)
+ df_upper = _df.loc[_df.index.get_level_values('stat') == 'upper'].reset_index(level='stat', drop=True)
+
+ # Create the scatter plot
+ plt.figure(figsize=(10, 6))
+
+ # Plot each point with error bars (for confidence interval)
+ plt.errorbar(df_mean[_calibration_var],
+ df_mean['model_cost'],
+ yerr=[df_mean['model_cost'] - df_lower['model_cost'], df_upper['model_cost'] - df_mean['model_cost']],
+ fmt='o',
+ ecolor='gray',
+ capsize=5,
+ label='Calibration Category')
+
+ # Adding the 45-degree line (where y = x)
+ min_val = min(df_mean[_calibration_var].min(), df_mean['model_cost'].min())
+ max_val = max(df_mean[_calibration_var].max(), df_mean['model_cost'].max())
+ plt.plot([min_val, max_val], [min_val, max_val], 'r--', label='45-degree line') # Red dashed line
+
+ # Add labels for each calibration_category
+ for i, label in enumerate(df_mean.index):
+ plt.annotate(label, (df_mean[_calibration_var].iloc[i], df_mean['model_cost'].iloc[i]))
+
+ # Add labels and title
+ plt.xlabel('Actual Expenditure 2019')
+ plt.ylabel('Model Cost (with confidence interval)')
+ plt.title(f'Model Cost vs {_calibration_var}')
+
+ # Show the plot
+ plt.tight_layout()
+ cost_subcategory = [name for name in globals() if globals()[name] is _costs_included][0]
+ cost_subcategory = cost_subcategory.replace('list_of_', '').replace('_for_calibration', '')
+ plt.savefig(calibration_outputs_folder / f'calibration_{_calibration_var}_{cost_subcategory}.png', dpi=100,
+ bbox_inches='tight')
+ plt.close()
+
+for var in ['actual_expenditure_2019', 'max_annual_budget_2020-22']:
+ do_cost_calibration_plot(calibration_data, list_of_consumables_costs_for_calibration_only_hiv, var)
+ do_cost_calibration_plot(calibration_data, list_of_consumables_costs_for_calibration_without_hiv, var)
+ do_cost_calibration_plot(calibration_data, list_of_hr_costs_for_calibration, var)
+ do_cost_calibration_plot(calibration_data, list_of_equipment_costs_for_calibration, var)
+
+
+'''
diff --git a/src/scripts/costing/example_costing_scenario.py b/src/scripts/costing/example_costing_scenario.py
new file mode 100644
index 0000000000..30ca966f96
--- /dev/null
+++ b/src/scripts/costing/example_costing_scenario.py
@@ -0,0 +1,122 @@
+'''
+Run on the batch system using:
+```tlo batch-submit src/scripts/costing/example_costing_scenario.py```
+
+or locally using:
+ ```tlo scenario-run src/scripts/costing/example_costing_scenario.py```
+
+'''
+
+from pathlib import Path
+from typing import Dict
+
+from tlo import Date, logging
+from tlo.analysis.utils import get_parameters_for_status_quo, mix_scenarios
+from tlo.methods.fullmodel import fullmodel
+from tlo.methods.scenario_switcher import ImprovedHealthSystemAndCareSeekingScenarioSwitcher
+from tlo.scenario import BaseScenario
+
+class CostingScenarios(BaseScenario):
+ def __init__(self):
+ super().__init__()
+ self.seed = 0
+ self.start_date = Date(2010, 1, 1)
+ self.end_date = Date(2030, 1, 1)
+ self.pop_size = 1_000 # <- recommended population size for the runs
+ self._scenarios = self._get_scenarios()
+ self.number_of_draws = len(self._scenarios)
+ self.runs_per_draw = 2 # <- repeated this many times
+
+ def log_configuration(self):
+ return {
+ 'filename': 'cost_scenarios',
+ 'directory': './outputs', # <- (specified only for local running)
+ 'custom_levels': {
+ '*': logging.WARNING,
+ 'tlo.methods.demography': logging.INFO,
+ 'tlo.methods.healthburden': logging.INFO,
+ 'tlo.methods.healthsystem.summary': logging.INFO,
+ }
+ }
+
+ def modules(self):
+ return (fullmodel(resourcefilepath=self.resources) +
+ [ImprovedHealthSystemAndCareSeekingScenarioSwitcher(resourcefilepath=self.resources)])
+
+ def draw_parameters(self, draw_number, rng):
+ if draw_number < len(self._scenarios):
+ return list(self._scenarios.values())[draw_number]
+
+ def _get_scenarios(self) -> Dict[str, Dict]:
+ """Return the Dict with values for the parameters that are changed, keyed by a name for the scenario."""
+
+ self.YEAR_OF_SYSTEM_CHANGE = 2020
+ self.mode_appt_constraints_postSwitch = [1,2]
+ self.cons_availability = ['default', 'all']
+ self.healthsystem_function = [[False, False], [False, True]]
+ self.healthcare_seeking = [[False, False], [False, True]]
+
+ return {
+ "Real world": self._common_baseline(),
+
+ "Perfect health system":
+ mix_scenarios(
+ self._common_baseline(),
+ {
+ 'HealthSystem': {
+ # Human Resources
+ 'mode_appt_constraints_postSwitch': self.mode_appt_constraints_postSwitch[1], # <-- Mode 2 post-change to show effects of HRH
+ "scale_to_effective_capabilities": True, # <-- Transition into Mode2 with the effective capabilities in HRH 'revealed' in Mode 1
+ "year_mode_switch": self.YEAR_OF_SYSTEM_CHANGE,
+
+ # Consumables
+ 'cons_availability_postSwitch': self.cons_availability[1],
+ 'year_cons_availability_switch': self.YEAR_OF_SYSTEM_CHANGE,
+ },
+ 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': {
+ 'max_healthcare_seeking': self.healthcare_seeking[1],
+ 'max_healthsystem_function': self.healthsystem_function[1],
+ 'year_of_switch': self.YEAR_OF_SYSTEM_CHANGE,
+ }
+ }
+ ),
+ }
+
+ def _common_baseline(self) -> Dict:
+ """Return the Dict with values for the parameter changes that define the baseline scenario. """
+ return mix_scenarios(
+ get_parameters_for_status_quo(), # <-- Parameters that have been the calibration targets
+ # Set up the HealthSystem to transition from Mode 1 -> Mode 2, with rescaling when there are HSS changes
+ {
+ 'HealthSystem': {
+ # Human resources
+ 'mode_appt_constraints': 1, # <-- Mode 1 prior to change to preserve calibration
+ 'mode_appt_constraints_postSwitch': self.mode_appt_constraints_postSwitch[0], # <-- Mode 2 post-change to show effects of HRH
+ "scale_to_effective_capabilities": True, # <-- Transition into Mode2 with the effective capabilities in HRH 'revealed' in Mode 1
+ # This happens in the year before mode change, as the model calibration is done by that year
+ "year_mode_switch": self.YEAR_OF_SYSTEM_CHANGE,
+ 'yearly_HR_scaling_mode': 'historical_scaling', # for 5 years of 2020-2024; source data year 2019
+
+ # Consumables
+ 'cons_availability': 'default',
+ 'cons_availability_postSwitch': self.cons_availability[0],
+ 'year_cons_availability_switch': self.YEAR_OF_SYSTEM_CHANGE,
+
+ # Normalize the behaviour of Mode 2
+ "policy_name": 'Naive',
+ "tclose_overwrite": 1,
+ "tclose_days_offset_overwrite": 7,
+ },
+ 'ImprovedHealthSystemAndCareSeekingScenarioSwitcher': {
+ 'max_healthcare_seeking': self.healthcare_seeking[0],
+ 'max_healthsystem_function': self.healthsystem_function[0],
+ 'year_of_switch': self.YEAR_OF_SYSTEM_CHANGE,
+ }
+ },
+ )
+
+
+if __name__ == '__main__':
+ from tlo.cli import scenario_run
+
+ scenario_run([__file__])
diff --git a/src/scripts/data_file_processing/healthsystem/consumables/consumable_resource_analyses_with_lmis/consumables_availability_estimation.py b/src/scripts/data_file_processing/healthsystem/consumables/consumable_resource_analyses_with_lmis/consumables_availability_estimation.py
index 3615afd400..2495ea6d66 100644
--- a/src/scripts/data_file_processing/healthsystem/consumables/consumable_resource_analyses_with_lmis/consumables_availability_estimation.py
+++ b/src/scripts/data_file_processing/healthsystem/consumables/consumable_resource_analyses_with_lmis/consumables_availability_estimation.py
@@ -277,10 +277,14 @@ def custom_agg(x):
return _collapsed_df
# Hold out the dataframe with no naming inconsistencies
-list_of_items_with_inconsistent_names_zipped = set(zip(inconsistent_item_names_mapping.keys(), inconsistent_item_names_mapping.values()))
-list_of_items_with_inconsistent_names = [item for sublist in list_of_items_with_inconsistent_names_zipped for item in sublist]
-df_with_consistent_item_names = lmis_df_wide_flat[~lmis_df_wide_flat[('item',)].isin(list_of_items_with_inconsistent_names)]
-df_without_consistent_item_names = lmis_df_wide_flat[lmis_df_wide_flat[('item',)].isin(list_of_items_with_inconsistent_names)]
+list_of_items_with_inconsistent_names_zipped = list(
+ zip(inconsistent_item_names_mapping.keys(), inconsistent_item_names_mapping.values()))
+list_of_items_with_inconsistent_names = [
+ item for sublist in list_of_items_with_inconsistent_names_zipped for item in sublist]
+df_with_consistent_item_names = lmis_df_wide_flat[~lmis_df_wide_flat[('item',)].isin(
+ list_of_items_with_inconsistent_names)]
+df_without_consistent_item_names = lmis_df_wide_flat[lmis_df_wide_flat[('item',)].isin(
+ list_of_items_with_inconsistent_names)]
# Make inconsistently named drugs uniform across the dataframe
df_without_consistent_item_names_corrected = rename_items_to_address_inconsistentencies(
df_without_consistent_item_names, inconsistent_item_names_mapping)
diff --git a/src/scripts/data_file_processing/healthsystem/consumables/processing_data_from_one_health/generate_consumables_item_codes_and_packages.py b/src/scripts/data_file_processing/healthsystem/consumables/processing_data_from_one_health/generate_consumables_item_codes_and_packages.py
index 7ca04f763f..3fcbccf9e2 100644
--- a/src/scripts/data_file_processing/healthsystem/consumables/processing_data_from_one_health/generate_consumables_item_codes_and_packages.py
+++ b/src/scripts/data_file_processing/healthsystem/consumables/processing_data_from_one_health/generate_consumables_item_codes_and_packages.py
@@ -21,8 +21,7 @@
# Set local Dropbox source
path_to_dropbox = Path( # <-- point to the TLO dropbox locally
- # '/Users/tbh03/Dropbox (SPH Imperial College)/Thanzi la Onse Theme 1 SHARE'
- '/Users/sm2511/Dropbox/Thanzi La Onse')
+ '/Users/tbh03/Dropbox (SPH Imperial College)/Thanzi la Onse Theme 1 SHARE')
resourcefilepath = Path("./resources")
path_for_new_resourcefiles = resourcefilepath / "healthsystem/consumables"
@@ -246,7 +245,7 @@
def add_record(df: pd.DataFrame, record: Dict):
"""Add a row to the bottom of the dataframe, where the row is specified by a dict keyed by the target columns."""
- assert set(df.columns) == set(record.keys())
+ assert list(df.columns) == list(record.keys())
return pd.concat([df, pd.DataFrame.from_records([record])], ignore_index=True)
@@ -329,54 +328,6 @@ def add_record(df: pd.DataFrame, record: Dict):
},
)
-cons = add_record(
- cons,
- {
- 'Intervention_Cat': "Added by SM (Recommended by TM)",
- 'Intervention_Pkg': "Isoniazid preventative therapy for HIV+ no TB",
- 'Intervention_Pkg_Code': 82,
- 'Items': "Isoniazid/Rifapentine",
- 'Item_Code': 2678,
- 'Expected_Units_Per_Case': 1.0,
- 'Unit_Cost': 1.0
- },
-)
-
-cons = add_record(
- cons,
- {
- 'Intervention_Cat': "Added by SM (Recommended by EJ)",
- 'Intervention_Pkg': "Misc",
- 'Intervention_Pkg_Code': -99,
- 'Items': "Cystoscope",
- 'Item_Code': 285,
- 'Expected_Units_Per_Case': 1.0,
- 'Unit_Cost': np.nan},
-)
-
-cons = add_record(
- cons,{
- 'Intervention_Cat': "Added by SM (Recommended by EJ)",
- 'Intervention_Pkg': "Misc",
- 'Intervention_Pkg_Code': -99,
- 'Items': "Endoscope",
- 'Item_Code': 280,
- 'Expected_Units_Per_Case': 1.0,
- 'Unit_Cost': np.nan},
-)
-
-cons = add_record(
- cons,{
- 'Intervention_Cat': "Added by SM (Recommended by EJ)",
- 'Intervention_Pkg': "Misc",
- 'Intervention_Pkg_Code': -99,
- 'Items': "Prostate specific antigen test",
- 'Item_Code': 281,
- 'Expected_Units_Per_Case': 1.0,
- 'Unit_Cost': np.nan},
-)
-
-
# --------------
# --------------
# --------------
diff --git a/src/scripts/data_file_processing/healthsystem/equipment/equipment_availability_estimation.py b/src/scripts/data_file_processing/healthsystem/equipment/equipment_availability_estimation.py
index 12ba3c7f9d..0a742d37b8 100644
--- a/src/scripts/data_file_processing/healthsystem/equipment/equipment_availability_estimation.py
+++ b/src/scripts/data_file_processing/healthsystem/equipment/equipment_availability_estimation.py
@@ -322,12 +322,6 @@
.drop_duplicates() \
.pipe(lambda x: x.set_index(x['Item_code'].astype(int)))['Category'] \
.to_dict()
-# Manually declare the price category for equipment items added manually
-# 402: Endoscope: 'Cost >= $1000'
-equipment_price_category_mapper[402] = 'Cost >= $1000'
-# 403: Electrocardiogram: 'Cost >= $1000'
-equipment_price_category_mapper[403] = 'Cost >= $1000'
-
equipment_price_category = final_equipment_availability_export_full.index.get_level_values('Item_Code') \
.map(equipment_price_category_mapper)
final_equipment_availability_export_full = final_equipment_availability_export_full.groupby(
diff --git a/src/scripts/data_file_processing/healthsystem/health_facilities/get_number_of_facilities.py b/src/scripts/data_file_processing/healthsystem/health_facilities/get_number_of_facilities.py
new file mode 100644
index 0000000000..0522580dbe
--- /dev/null
+++ b/src/scripts/data_file_processing/healthsystem/health_facilities/get_number_of_facilities.py
@@ -0,0 +1,154 @@
+"""
+This script extracts the number of health facilities by level and district from the HHFA 2018-19
+
+Inputs:
+1. Raw HHFA data - Q1.dta (~Dropbox/Thanzi la Onse/07 - Data/HHFA_2018-19/0 raw/2_Final data/)
+2. Cleaned variable names for HHFA data - variable_list.csv (~Dropbox/Thanzi la Onse/07 - Data/HHFA_2018-19/1 processing)
+
+Outputs:
+1. updated master facilities list resource file - ResourceFile_Master_Facilities_List.csv
+"""
+
+import calendar
+import datetime
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+from tabulate import tabulate
+import copy
+
+# Set local Dropbox source
+path_to_dropbox = Path( # <-- point to the TLO dropbox locally
+ '/Users/sm2511/Dropbox/Thanzi la Onse'
+)
+
+path_to_files_in_the_tlo_dropbox = path_to_dropbox / "07 - Data/HHFA_2018-19/" # <-- point to HHFA data folder in dropbox
+resourcefilepath = Path("./resources")
+
+# define a timestamp for script outputs
+timestamp = datetime.datetime.now().strftime("_%Y_%m_%d_%H_%M")
+
+# print the start time of the script
+print('Script Start', datetime.datetime.now().strftime('%H:%M'))
+
+# %%
+## 1. DATA IMPORT ##
+raw_hhfa = pd.read_csv(path_to_files_in_the_tlo_dropbox / '0 raw/2_Final data/Q1.csv',
+ low_memory=False) # import 2018 data
+varnames = pd.read_csv(path_to_files_in_the_tlo_dropbox / '1 processing/variable_list.csv',
+ encoding="ISO-8859-1") # import file with cleaned variable names
+
+# Rename HHFA columns using variable name mapping in loaded .csv
+old_var_name = varnames['var']
+new_var_name = varnames['new_var_name']
+
+hhfa = copy.deepcopy(raw_hhfa)
+for i in range(len(new_var_name)):
+ if new_var_name[i] != np.nan:
+ hhfa.rename(columns={old_var_name[i]: new_var_name[i]},
+ inplace=True)
+ else:
+ pass
+
+# Rename columns with missing data to "a" and then drop these columns since these will not be used in the analysis
+hhfa.rename({np.nan: "a"}, axis="columns", inplace=True)
+hhfa.drop(["a"], axis=1, inplace=True)
+
+# Preserve only relevant columns
+facility_identification_columns = ['fac_code', 'fac_name', 'region', 'zone','district', 'fac_type', 'fac_location', 'fac_owner']
+hhfa = hhfa[facility_identification_columns]
+
+# %%
+## 2. FEATURE CLEANING ##
+# Clean district names #
+hhfa.loc[hhfa['district'] == 'Blanytyre', 'district'] = 'Blantyre'
+hhfa.loc[hhfa['district'] == 'Nkhatabay', 'district'] = 'Nkhata Bay'
+
+# Pvt for profit hospital incorrectly labelled District Hospital
+cond = hhfa.fac_code == 5067
+hhfa.loc[cond, 'fac_type'] = 'Other Hospital'
+
+# Clean fac_owner
+cond = hhfa.fac_owner == 'Private non profit'
+hhfa.loc[cond, 'fac_owner'] = 'NGO'
+
+# convert fac_location to binary (Yes/No)
+hhfa = hhfa.rename(columns={'fac_location': 'fac_urban'})
+cond1 = hhfa.fac_urban.str.lower() == "rural"
+hhfa.loc[cond1, 'fac_urban'] = 0
+cond2 = hhfa.fac_urban.str.lower() == "urban"
+hhfa.loc[cond2, 'fac_urban'] = 1
+
+# Clean facility type
+hhfa['fac_type'] = hhfa['fac_type'].str.replace(' ', '').str.lower()
+hhfa['Facility_Level'] = ""
+
+def assign_facilty_level_based_on_hhfa_facility_names(_df):
+ cond_mch = (_df['fac_name'].str.replace(' ', '').str.lower().str.contains('mzuzucent'))
+ _df.loc[cond_mch, 'fac_name'] = 'Mzuzu Central Hospital'
+ cond_level0 = (_df['fac_name'].str.replace(' ', '').str.lower().str.contains('healthpost')) | \
+ (_df['fac_type'].str.contains('healthpost'))
+ cond_level1a = (_df['fac_type'] == 'clinic') | (_df['fac_type'] == 'healthcentre') | \
+ (_df['fac_type'].str.replace(' ', '').str.lower().str.contains('dispensary')) | \
+ (_df['fac_type'].str.replace(' ', '').str.lower().str.contains('maternity'))
+ cond_level1b = (_df['fac_type'].str.contains('communityhospital')) | \
+ (_df['fac_type'] == 'otherhospital')
+ cond_level2 = (_df['fac_type'] == 'districthospital')
+ cond_level3 = _df.fac_name.str.replace(' ', '').str.lower().str.contains("centralhospit")
+ cond_level4 = _df.fac_name.str.replace(' ', '').str.lower().str.contains("mentalhospit")
+
+ _df.loc[cond_level0,'Facility_Level'] = '0'
+ _df.loc[cond_level1a,'Facility_Level'] = '1a'
+ _df.loc[cond_level1b,'Facility_Level'] = '1b'
+ _df.loc[cond_level2,'Facility_Level'] = '2'
+ _df.loc[cond_level3,'Facility_Level'] = '3'
+ _df.loc[cond_level4,'Facility_Level'] = '4'
+
+assign_facilty_level_based_on_hhfa_facility_names(hhfa)
+hhfa = hhfa.drop_duplicates('fac_name')
+
+# Count facilities by category
+# Count number of private facilities by district
+cond_private = hhfa.fac_owner.str.contains("Private")
+cond_level0 = hhfa.Facility_Level == '0'
+private_facility_count = hhfa[cond_private & ~cond_level0].groupby('district')['fac_name'].count()
+
+# Count number of NGO facilities by district
+cond_ngo = hhfa.fac_owner.str.contains("NGO")
+ngo_facility_count = hhfa[cond_ngo & ~cond_level0].groupby('district')['fac_name'].count()
+
+# For the TLO model, we are only concerned with government and CHAM facilities
+tlo_model_facilities = hhfa[~(cond_ngo|cond_private)]
+facility_count_govt_and_cham = tlo_model_facilities.groupby(['district', 'Facility_Level'])['fac_name'].count().reset_index()
+# Collapse data for Mzimba North and South into 'Mzimba'
+cond_north = facility_count_govt_and_cham['district'] == 'Mzimba North'
+cond_south = facility_count_govt_and_cham['district'] == 'Mzimba South'
+facility_count_govt_and_cham.loc[(cond_north|cond_south), 'district'] = 'Mzimba'
+facility_count_govt_and_cham = facility_count_govt_and_cham.groupby(['district', 'Facility_Level']).sum()
+
+tlo_model_facilities['govt'] = 0
+tlo_model_facilities.loc[tlo_model_facilities.fac_owner == "Government", 'govt'] = 1
+proportion_of_facilities_run_by_govt = tlo_model_facilities.groupby(['district', 'Facility_Level'])['govt'].mean()
+
+proportion_of_facilities_in_urban_location = tlo_model_facilities.groupby(['district', 'Facility_Level'])['fac_urban'].mean()
+
+facility_count_data = pd.merge(facility_count_govt_and_cham, proportion_of_facilities_run_by_govt, right_index=True, left_index=True, how = 'left', validate = "1:1")
+facility_count_data = pd.merge(facility_count_data, proportion_of_facilities_in_urban_location, right_index=True, left_index=True, how = 'left', validate = "1:1")
+facility_count_data = facility_count_data.reset_index().rename(columns = {'district' : 'District',
+ 'fac_name' : 'Facility_Count',
+ 'govt': 'Proportion_owned_by_government',
+ 'fac_urban': 'Proportion_located_in_urban_area'})
+facility_count_data = facility_count_data[~(facility_count_data.Facility_Level.isin(['3', '4', '5']))]
+
+#%%
+# Add this data to the Master Health Facilities Resource File
+mfl = pd.read_csv(resourcefilepath / "healthsystem" / "organisation" / "ResourceFile_Master_Facilities_List.csv")[['District', 'Facility_Level', 'Region', 'Facility_ID','Facility_Name']]
+mfl = mfl.merge(facility_count_data, on = ['District', 'Facility_Level'], how = 'left')
+mfl.loc[mfl.Facility_Level.isin(['3', '4', '5']), 'Facility_Count'] = 1
+mfl.loc[mfl.Facility_Level.isin(['3', '4', '5']), 'Proportion_owned_by_government'] = 1
+mfl.loc[mfl.Facility_Count.isna(), 'Facility_Count'] = 0
+
+# Export Master Health Facilities Resource File with facility count data
+mfl.to_csv(resourcefilepath / "healthsystem" / "organisation" / "ResourceFile_Master_Facilities_List.csv", index = False)
diff --git a/src/scripts/dependencies/tlo_module_graph.py b/src/scripts/dependencies/tlo_module_graph.py
deleted file mode 100644
index 278539db31..0000000000
--- a/src/scripts/dependencies/tlo_module_graph.py
+++ /dev/null
@@ -1,82 +0,0 @@
-"""Construct a graph showing dependencies between modules."""
-
-import argparse
-from pathlib import Path
-from typing import Dict, Set
-
-from tlo.dependencies import DependencyGetter, get_all_dependencies, get_module_class_map
-from tlo.methods import Metadata
-
-try:
- import pydot
-except ImportError:
- pydot = None
-
-
-def construct_module_dependency_graph(
- excluded_modules: Set[str],
- disease_module_node_defaults: Dict,
- other_module_node_defaults: Dict,
- get_dependencies: DependencyGetter = get_all_dependencies,
-):
- """Construct a pydot object representing module dependency graph.
-
- :param excluded_modules: Set of ``Module`` subclass names to not included in graph.
- :param disease_module_node_defaults: Any dot node attributes to apply to by default
- to disease module nodes.
- :param other_module_node_defaults: Any dot node attributes to apply to by default
- to non-disease module nodes.
- :param get_dependencies: Function which given a module gets the set of module
- dependencies. Defaults to extracting all dependencies.
- :return: Pydot directed graph representing module dependencies.
- """
- if pydot is None:
- raise RuntimeError("pydot package must be installed")
- module_class_map = get_module_class_map(excluded_modules)
- module_graph = pydot.Dot("modules", graph_type="digraph")
- disease_module_subgraph = pydot.Subgraph("disease_modules")
- module_graph.add_subgraph(disease_module_subgraph)
- other_module_subgraph = pydot.Subgraph("other_modules")
- module_graph.add_subgraph(other_module_subgraph)
- disease_module_subgraph.set_node_defaults(**disease_module_node_defaults)
- other_module_subgraph.set_node_defaults(**other_module_node_defaults)
- for name, module_class in module_class_map.items():
- node = pydot.Node(name)
- if Metadata.DISEASE_MODULE in module_class.METADATA:
- disease_module_subgraph.add_node(node)
- else:
- other_module_subgraph.add_node(node)
- for key, module in module_class_map.items():
- for dependency in get_dependencies(module, module_class_map.keys()):
- if dependency not in excluded_modules:
- module_graph.add_edge(pydot.Edge(key, dependency))
- return module_graph
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument(
- "output_file", type=Path, help=(
- "Path to output graph to. File extension will determine output format - for example: dot, dia, png, svg"
- )
- )
- args = parser.parse_args()
- excluded_modules = {
- "Mockitis",
- "ChronicSyndrome",
- "Skeleton",
- "AlriPropertiesOfOtherModules",
- "DiarrhoeaPropertiesOfOtherModules",
- "DummyHivModule",
- "SimplifiedBirths",
- "Tb",
- }
- module_graph = construct_module_dependency_graph(
- excluded_modules,
- disease_module_node_defaults={"fontname": "Arial", "shape": "box"},
- other_module_node_defaults={"fontname": "Arial", "shape": "ellipse"},
- )
- format = (
- args.output_file.suffix[1:] if args.output_file.suffix else "raw"
- )
- module_graph.write(args.output_file, format=format)
diff --git a/src/scripts/epilepsy_analyses/analysis_epilepsy.py b/src/scripts/epilepsy_analyses/analysis_epilepsy.py
index bba4d3c479..735cbc6ce7 100644
--- a/src/scripts/epilepsy_analyses/analysis_epilepsy.py
+++ b/src/scripts/epilepsy_analyses/analysis_epilepsy.py
@@ -28,7 +28,7 @@
start_date = Date(2010, 1, 1)
end_date = Date(2020, 1, 1)
-popsize = 100_000
+popsize = 200000
# Establish the simulation object
log_config = {
@@ -40,11 +40,10 @@
'tlo.methods.demography': logging.INFO,
'tlo.methods.healthsystem': logging.WARNING,
'tlo.methods.healthburden': logging.WARNING,
- 'tlo.methods.population': logging.INFO,
}
}
-sim = Simulation(start_date=start_date, seed=0, log_config=log_config, show_progress_bar=True)
+sim = Simulation(start_date=start_date, seed=0, log_config=log_config)
# make a dataframe that contains the switches for which interventions are allowed or not allowed
# during this run. NB. These must use the exact 'registered strings' that the disease modules allow
@@ -126,8 +125,7 @@
)
n_seiz_stat_1_3.plot()
plt.title('Number with epilepsy (past or current)')
-plt.gca().set_ylim(bottom=0)
-plt.ylabel("Number (not scaled)")
+plt.ylim(0, 800000)
plt.tight_layout()
plt.show()
@@ -137,25 +135,11 @@
)
n_seiz_stat_2_3.plot()
plt.title('Number with epilepsy (infrequent or frequent seizures)')
-plt.gca().set_ylim(bottom=0)
-plt.ylabel("Number (not scaled)")
+plt.ylim(0, 300000)
plt.tight_layout()
plt.show()
plt.clf()
-
-prop_antiepilep_seiz_infreq_or_freq = pd.Series(
- output['tlo.methods.epilepsy']['epilepsy_logging']['prop_freq_or_infreq_seiz_on_antiep'].values,
- index=output['tlo.methods.epilepsy']['epilepsy_logging']['date']
-)
-prop_antiepilep_seiz_infreq_or_freq.plot(color='r')
-plt.title('Proportion on antiepileptics\namongst people that have infrequent or frequent epileptic seizures')
-plt.ylim(0, 1)
-plt.tight_layout()
-plt.show()
-plt.clf()
-
-
prop_antiepilep_seiz_stat_1 = pd.Series(
output['tlo.methods.epilepsy']['epilepsy_logging']['prop_antiepilep_seiz_stat_1'].values,
index=output['tlo.methods.epilepsy']['epilepsy_logging']['date']
@@ -195,8 +179,7 @@
)
n_epi_death.plot()
plt.title('Number of deaths from epilepsy')
-plt.gca().set_ylim(bottom=0)
-plt.ylabel("Number (not scaled)")
+plt.ylim(0, 50)
plt.tight_layout()
plt.show()
plt.clf()
@@ -207,21 +190,11 @@
)
n_antiep.plot()
plt.title('Number of people on antiepileptics')
-plt.gca().set_ylim(bottom=0)
-plt.ylabel("Number (not scaled)")
+plt.ylim(0, 50000)
plt.tight_layout()
plt.show()
plt.clf()
-(n_antiep / popsize).plot()
-plt.title('Proportion of of people (whole population) on antiepileptics')
-plt.gca().set_ylim(bottom=0)
-plt.ylabel("Number (not scaled)")
-plt.tight_layout()
-plt.show()
-plt.clf()
-
-
epi_death_rate = pd.Series(
output['tlo.methods.epilepsy']['epilepsy_logging']['epi_death_rate'].values,
index=output['tlo.methods.epilepsy']['epilepsy_logging']['date']
@@ -260,7 +233,8 @@
for _row, period in enumerate(('2010-2014', '2015-2019')):
ax = axs[_row]
comparison.loc[(period, slice(None), slice(None), CAUSE_NAME)]\
- .groupby(axis=0, level=1)\
+ .droplevel([0, 1, 3])\
+ .groupby(axis=0, level=0)\
.sum()\
.plot(use_index=True, ax=ax)
ax.set_ylabel('Deaths per year')
diff --git a/src/scripts/hiv/projections_jan2023/analysis_logged_deviance.py b/src/scripts/hiv/projections_jan2023/analysis_logged_deviance.py
index 7a2af7fbed..eca9f999bc 100644
--- a/src/scripts/hiv/projections_jan2023/analysis_logged_deviance.py
+++ b/src/scripts/hiv/projections_jan2023/analysis_logged_deviance.py
@@ -34,8 +34,8 @@
# %% Run the simulation
start_date = Date(2010, 1, 1)
-end_date = Date(2022, 1, 1)
-popsize = 5000
+end_date = Date(2014, 1, 1)
+popsize = 1000
# scenario = 1
@@ -87,8 +87,8 @@
)
# set the scenario
-sim.modules["Hiv"].parameters["do_scaleup"] = True
-sim.modules["Hiv"].parameters["scaleup_start_year"] = 2019
+# sim.modules["Hiv"].parameters["beta"] = 0.129671
+# sim.modules["Tb"].parameters["scaling_factor_WHO"] = 1.5
# sim.modules["Tb"].parameters["scenario"] = scenario
# sim.modules["Tb"].parameters["scenario_start_date"] = Date(2010, 1, 1)
# sim.modules["Tb"].parameters["scenario_SI"] = "z"
diff --git a/src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py b/src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py
deleted file mode 100644
index beacb5e218..0000000000
--- a/src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py
+++ /dev/null
@@ -1,112 +0,0 @@
-
-"""
-This scenario file sets up the scenarios for simulating the effects of scaling up programs
-
-The scenarios are:
-*0 baseline mode 1
-*1 scale-up HIV program
-*2 scale-up TB program
-*3 scale-up malaria program
-*4 scale-up HIV and Tb and malaria programs
-
-scale-up occurs on the default scale-up start date (01/01/2025: in parameters list of resourcefiles)
-
-For all scenarios, keep all default health system settings
-
-check the batch configuration gets generated without error:
-tlo scenario-run --draw-only src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py
-
-Run on the batch system using:
-tlo batch-submit src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py
-
-or locally using:
-tlo scenario-run src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py
-
-or execute a single run:
-tlo scenario-run src/scripts/htm_scenario_analyses/analysis_htm_scaleup.py --draw 1 0
-
-"""
-
-from pathlib import Path
-
-from tlo import Date, logging
-from tlo.methods import (
- demography,
- enhanced_lifestyle,
- epi,
- healthburden,
- healthseekingbehaviour,
- healthsystem,
- hiv,
- malaria,
- simplified_births,
- symptommanager,
- tb,
-)
-from tlo.scenario import BaseScenario
-
-
-class EffectOfProgrammes(BaseScenario):
- def __init__(self):
- super().__init__()
- self.seed = 0
- self.start_date = Date(2010, 1, 1)
- self.end_date = Date(2025, 1, 1)
- self.pop_size = 5_000
- self.number_of_draws = 2
- self.runs_per_draw = 1
-
- def log_configuration(self):
- return {
- 'filename': 'scaleup_tests',
- 'directory': Path('./outputs'), # <- (specified only for local running)
- 'custom_levels': {
- '*': logging.WARNING,
- 'tlo.methods.hiv': logging.INFO,
- 'tlo.methods.tb': logging.INFO,
- 'tlo.methods.malaria': logging.INFO,
- 'tlo.methods.demography': logging.INFO,
- }
- }
-
- def modules(self):
-
- return [
- demography.Demography(resourcefilepath=self.resources),
- simplified_births.SimplifiedBirths(resourcefilepath=self.resources),
- enhanced_lifestyle.Lifestyle(resourcefilepath=self.resources),
- healthsystem.HealthSystem(resourcefilepath=self.resources),
- symptommanager.SymptomManager(resourcefilepath=self.resources),
- healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=self.resources),
- healthburden.HealthBurden(resourcefilepath=self.resources),
- epi.Epi(resourcefilepath=self.resources),
- hiv.Hiv(resourcefilepath=self.resources),
- tb.Tb(resourcefilepath=self.resources),
- malaria.Malaria(resourcefilepath=self.resources),
- ]
-
- def draw_parameters(self, draw_number, rng):
- scaleup_start_year = 2019
-
- return {
- 'Hiv': {
- 'do_scaleup': [False, True, False, False, True][draw_number],
- 'scaleup_start_year': scaleup_start_year
- },
- 'Tb': {
- 'do_scaleup': [False, False, True, False, True][draw_number],
- 'scaleup_start_year': scaleup_start_year
- },
- 'Malaria': {
- 'do_scaleup': [False, False, False, True, True][draw_number],
- 'scaleup_start_year': scaleup_start_year
- },
- }
-
-
-if __name__ == '__main__':
- from tlo.cli import scenario_run
-
- scenario_run([__file__])
-
-
diff --git a/src/scripts/htm_scenario_analyses/scenario_plots.py b/src/scripts/htm_scenario_analyses/scenario_plots.py
deleted file mode 100644
index c209c60f6e..0000000000
--- a/src/scripts/htm_scenario_analyses/scenario_plots.py
+++ /dev/null
@@ -1,140 +0,0 @@
-""" this reads in the outputs generates through analysis_htm_scaleup.py
-and produces plots for HIV, TB and malaria incidence
-"""
-
-
-import datetime
-from pathlib import Path
-
-import matplotlib.pyplot as plt
-import pandas as pd
-import seaborn as sns
-
-from tlo import Date
-from tlo.analysis.utils import (
- extract_params,
- extract_results,
- get_scenario_info,
- get_scenario_outputs,
- load_pickled_dataframes,
-)
-
-resourcefilepath = Path("./resources")
-datestamp = datetime.date.today().strftime("__%Y_%m_%d")
-
-outputspath = Path("./outputs")
-# outputspath = Path("./outputs/t.mangal@imperial.ac.uk")
-
-
-# 0) Find results_folder associated with a given batch_file (and get most recent [-1])
-results_folder = get_scenario_outputs("scaleup_tests", outputspath)[-1]
-
-# Declare path for output graphs from this script
-make_graph_file_name = lambda stub: results_folder / f"{stub}.png" # noqa: E731
-
-# look at one log (so can decide what to extract)
-log = load_pickled_dataframes(results_folder, draw=1)
-
-# get basic information about the results
-info = get_scenario_info(results_folder)
-
-# 1) Extract the parameters that have varied over the set of simulations
-params = extract_params(results_folder)
-
-
-# DEATHS
-
-
-def get_num_deaths_by_cause_label(_df):
- """Return total number of Deaths by label within the TARGET_PERIOD
- values are summed for all ages
- df returned: rows=COD, columns=draw
- """
- return _df \
- .loc[pd.to_datetime(_df.date).between(*TARGET_PERIOD)] \
- .groupby(_df['label']) \
- .size()
-
-
-TARGET_PERIOD = (Date(2020, 1, 1), Date(2025, 1, 1))
-
-num_deaths_by_cause_label = extract_results(
- results_folder,
- module='tlo.methods.demography',
- key='death',
- custom_generate_series=get_num_deaths_by_cause_label,
- do_scaling=False
- )
-
-
-def summarise_deaths_for_one_cause(results_folder, label):
- """ returns mean deaths for each year of the simulation
- values are aggregated across the runs of each draw
- for the specified cause
- """
-
- results_deaths = extract_results(
- results_folder,
- module="tlo.methods.demography",
- key="death",
- custom_generate_series=(
- lambda df: df.assign(year=df["date"].dt.year).groupby(
- ["year", "label"])["person_id"].count()
- ),
- do_scaling=True,
- )
- # removes multi-index
- results_deaths = results_deaths.reset_index()
-
- # select only cause specified
- tmp = results_deaths.loc[
- (results_deaths.label == label)
- ]
-
- # group deaths by year
- tmp = pd.DataFrame(tmp.groupby(["year"]).sum())
-
- # get mean for each draw
- mean_deaths = pd.concat({'mean': tmp.iloc[:, 1:].groupby(level=0, axis=1).mean()}, axis=1).swaplevel(axis=1)
-
- return mean_deaths
-
-
-aids_deaths = summarise_deaths_for_one_cause(results_folder, 'AIDS')
-tb_deaths = summarise_deaths_for_one_cause(results_folder, 'TB (non-AIDS)')
-malaria_deaths = summarise_deaths_for_one_cause(results_folder, 'Malaria')
-
-draw_labels = ['No scale-up', 'HIV, scale-up', 'TB scale-up', 'Malaria scale-up', 'HTM scale-up']
-
-colors = sns.color_palette("Set1", 5) # Blue, Orange, Green, Red
-
-
-# Create subplots
-fig, axs = plt.subplots(3, 1, figsize=(6, 10))
-
-# Plot for df1
-for i, col in enumerate(aids_deaths.columns):
- axs[0].plot(aids_deaths.index, aids_deaths[col], label=draw_labels[i], color=colors[i])
-axs[0].set_title('HIV/AIDS')
-axs[0].legend()
-axs[0].axvline(x=2019, color='gray', linestyle='--')
-
-# Plot for df2
-for i, col in enumerate(tb_deaths.columns):
- axs[1].plot(tb_deaths.index, tb_deaths[col], color=colors[i])
-axs[1].set_title('TB')
-axs[1].axvline(x=2019, color='gray', linestyle='--')
-
-# Plot for df3
-for i, col in enumerate(malaria_deaths.columns):
- axs[2].plot(malaria_deaths.index, malaria_deaths[col], color=colors[i])
-axs[2].set_title('Malaria')
-axs[2].axvline(x=2019, color='gray', linestyle='--')
-
-for ax in axs:
- ax.set_xlabel('Years')
- ax.set_ylabel('Number deaths')
-
-plt.tight_layout()
-plt.show()
-
diff --git a/src/scripts/malaria/analysis_malaria.py b/src/scripts/malaria/analysis_malaria.py
index b2b4217dc6..56d05cf3ae 100644
--- a/src/scripts/malaria/analysis_malaria.py
+++ b/src/scripts/malaria/analysis_malaria.py
@@ -34,8 +34,8 @@
resourcefilepath = Path("./resources")
start_date = Date(2010, 1, 1)
-end_date = Date(2014, 1, 1)
-popsize = 100
+end_date = Date(2016, 1, 1)
+popsize = 300
# set up the log config
@@ -84,15 +84,6 @@
)
)
-# update parameters
-sim.modules["Hiv"].parameters["do_scaleup"] = True
-sim.modules["Tb"].parameters["do_scaleup"] = True
-sim.modules["Malaria"].parameters["do_scaleup"] = True
-sim.modules["Hiv"].parameters["scaleup_start"] = 2
-sim.modules["Tb"].parameters["scaleup_start"] = 2
-sim.modules["Malaria"].parameters["scaleup_start"] = 2
-
-
# Run the simulation and flush the logger
sim.make_initial_population(n=popsize)
sim.simulate(end_date=end_date)
@@ -106,5 +97,5 @@
pickle.dump(dict(output), f, pickle.HIGHEST_PROTOCOL)
# load the results
-with open(outputpath / "malaria_run.pickle", "rb") as f:
+with open(outputpath / "default_run.pickle", "rb") as f:
output = pickle.load(f)
diff --git a/src/scripts/profiling/run_profiling.py b/src/scripts/profiling/run_profiling.py
index 6097177af9..882894d6af 100644
--- a/src/scripts/profiling/run_profiling.py
+++ b/src/scripts/profiling/run_profiling.py
@@ -12,7 +12,6 @@
from pyinstrument.renderers import ConsoleRenderer, HTMLRenderer
from pyinstrument.session import Session
from scale_run import save_arguments_to_json, scale_run
-from shared import memory_statistics
try:
from ansi2html import Ansi2HTMLConverter
@@ -169,8 +168,6 @@ def record_run_statistics(
**profiling_session_statistics(profiling_session),
# Disk input/output statistics
**disk_statistics(disk_usage),
- # Process memory statistics
- **memory_statistics(),
# Statistics from end end-state of the simulation
**simulation_statistics(completed_sim),
# User-defined additional stats (if any)
@@ -225,7 +222,7 @@ def run_profiling(
"initial_population": initial_population,
"log_filename": "scale_run_profiling",
"log_level": "WARNING",
- "parse_log_file": True,
+ "parse_log_file": False,
"show_progress_bar": show_progress_bar,
"seed": 0,
"disable_health_system": False,
@@ -248,7 +245,7 @@ def run_profiling(
# Profile scale_run
disk_at_start = disk_io_counters()
- completed_simulation, logs_dict = scale_run(
+ completed_simulation = scale_run(
**scale_run_args, output_dir=output_dir, profiler=profiler
)
disk_at_end = disk_io_counters()
@@ -326,13 +323,6 @@ def run_profiling(
additional_stats=additional_stats,
)
print("done")
-
- # Write out logged profiling statistics
- logged_statistics_file = output_dir / f"{output_name}.logged-stats.csv"
- print(f"Writing {logged_statistics_file}", end="...", flush=True)
- logs_dict["tlo.profiling"]["stats"].to_csv(logged_statistics_file, index=False)
- print("done")
-
if __name__ == "__main__":
diff --git a/src/scripts/profiling/scale_run.py b/src/scripts/profiling/scale_run.py
index 1e5d8042b3..735d1e7ba3 100644
--- a/src/scripts/profiling/scale_run.py
+++ b/src/scripts/profiling/scale_run.py
@@ -13,7 +13,6 @@
from shared import print_checksum, schedule_profile_log
from tlo import Date, Simulation, logging
-from tlo.analysis.utils import LogsDict
from tlo.analysis.utils import parse_log_file as parse_log_file_fn
from tlo.methods.fullmodel import fullmodel
@@ -56,10 +55,14 @@ def scale_run(
ignore_warnings: bool = False,
log_final_population_checksum: bool = True,
profiler: Optional["Profiler"] = None,
-) -> Simulation | tuple[Simulation, LogsDict]:
+) -> Simulation:
if ignore_warnings:
warnings.filterwarnings("ignore")
+ # Start profiler if one has been passed
+ if profiler is not None:
+ profiler.start()
+
# Simulation period
start_date = Date(2010, 1, 1)
end_date = start_date + pd.DateOffset(years=years, months=months)
@@ -67,14 +70,9 @@ def scale_run(
log_config = {
"filename": log_filename,
"directory": output_dir,
- # Ensure tlo.profiling log records always recorded
- "custom_levels": {"*": getattr(logging, log_level), "tlo.profiling": logging.INFO},
+ "custom_levels": {"*": getattr(logging, log_level)},
"suppress_stdout": disable_log_output_to_stdout,
}
-
- # Start profiler if one has been passed
- if profiler is not None:
- profiler.start()
sim = Simulation(
start_date=start_date,
@@ -104,19 +102,17 @@ def scale_run(
# Run the simulation
sim.make_initial_population(n=initial_population)
- schedule_profile_log(sim, frequency_months=1)
+ schedule_profile_log(sim)
sim.simulate(end_date=end_date)
-
- # Stop profiling session
- if profiler is not None:
- profiler.stop()
-
if log_final_population_checksum:
print_checksum(sim)
if save_final_population:
sim.population.props.to_pickle(output_dir / "final_population.pkl")
+ if parse_log_file:
+ parse_log_file_fn(sim.log_filepath)
+
if record_hsi_event_details:
with open(output_dir / "hsi_event_details.json", "w") as json_file:
json.dump(
@@ -128,11 +124,10 @@ def scale_run(
],
json_file,
)
-
- if parse_log_file:
- logs_dict = parse_log_file_fn(sim.log_filepath)
- return sim, logs_dict
+ # Stop profiling session
+ if profiler is not None:
+ profiler.stop()
return sim
diff --git a/src/scripts/profiling/shared.py b/src/scripts/profiling/shared.py
index caa06cf468..cc972cfa66 100644
--- a/src/scripts/profiling/shared.py
+++ b/src/scripts/profiling/shared.py
@@ -4,11 +4,6 @@
import pandas as pd
-try:
- import psutil
-except ImportError:
- psutil = None
-
from tlo import DateOffset, Simulation, logging
from tlo.events import PopulationScopeEventMixin, RegularEvent
from tlo.util import hash_dataframe
@@ -17,34 +12,9 @@
logger.setLevel(logging.INFO)
-def memory_statistics() -> dict[str, float]:
- """
- Extract memory usage statistics in current process using `psutil` if available.
- Statistics are returned as a dictionary. If `psutil` not installed an empty dict is returned.
-
- Key / value pairs are:
- memory_rss_MiB: float
- Resident set size in mebibytes. The non-swapped physical memory the process has used.
- memory_vms_MiB: float
- Virtual memory size in mebibytes. The total amount of virtual memory used by the process.
- memory_uss_MiB: float
- Unique set size in mebibytes. The memory which is unique to a process and which would be freed if the process
- was terminated right now
- """
- if psutil is None:
- return {}
- process = psutil.Process()
- memory_info = process.memory_full_info()
- return {
- "memory_rss_MiB": memory_info.rss / 2**20,
- "memory_vms_MiB": memory_info.vms / 2**20,
- "memory_uss_MiB": memory_info.uss / 2**20,
- }
-
-
class LogProgress(RegularEvent, PopulationScopeEventMixin):
- def __init__(self, module, frequency_months=3):
- super().__init__(module, frequency=DateOffset(months=frequency_months))
+ def __init__(self, module):
+ super().__init__(module, frequency=DateOffset(months=3))
self.time = time.time()
def apply(self, population):
@@ -56,18 +26,16 @@ def apply(self, population):
key="stats",
data={
"time": datetime.datetime.now().isoformat(),
- "duration_minutes": duration,
- "pop_df_number_alive": df.is_alive.sum(),
- "pop_df_rows": len(df),
- "pop_df_mem_MiB": df.memory_usage(index=True, deep=True).sum() / 2**20,
- **memory_statistics(),
+ "duration": duration,
+ "alive": df.is_alive.sum(),
+ "total": len(df),
},
)
-def schedule_profile_log(sim: Simulation, frequency_months: int = 3) -> None:
+def schedule_profile_log(sim: Simulation) -> None:
"""Schedules the log progress event, used only for profiling"""
- sim.schedule_event(LogProgress(sim.modules["Demography"], frequency_months), sim.start_date)
+ sim.schedule_event(LogProgress(sim.modules["Demography"]), sim.start_date)
def print_checksum(sim: Simulation) -> None:
diff --git a/src/tlo/analysis/hsi_events.py b/src/tlo/analysis/hsi_events.py
index 9bc973f67c..1a9d889ce4 100644
--- a/src/tlo/analysis/hsi_events.py
+++ b/src/tlo/analysis/hsi_events.py
@@ -47,7 +47,7 @@ def get_hsi_event_classes_per_module(
module = importlib.import_module(f'tlo.methods.{module_name}')
tlo_module_classes = [
obj for _, obj in inspect.getmembers(module)
- if is_valid_tlo_module_subclass(obj, {})
+ if is_valid_tlo_module_subclass(obj, excluded_modules)
]
hsi_event_classes = [
obj for _, obj in inspect.getmembers(module)
diff --git a/src/tlo/analysis/life_expectancy.py b/src/tlo/analysis/life_expectancy.py
index ebde940f66..6e3e9b4e83 100644
--- a/src/tlo/analysis/life_expectancy.py
+++ b/src/tlo/analysis/life_expectancy.py
@@ -99,36 +99,6 @@ def _aggregate_person_years_by_age(results_folder, target_period) -> pd.DataFram
return py_by_sex_and_agegroup
-def calculate_probability_of_dying(interval_width, fraction_of_last_age_survived, sex, _person_years_at_risk,
- _number_of_deaths_in_interval) -> pd.DataFrame:
- """Returns the probability of dying in each interval"""
-
- person_years_by_sex = _person_years_at_risk.xs(key=sex, level='sex')
-
- number_of_deaths_by_sex = _number_of_deaths_in_interval.xs(key=sex, level='sex')
-
- death_rate_in_interval = number_of_deaths_by_sex / person_years_by_sex
-
- death_rate_in_interval = death_rate_in_interval.fillna(0)
-
- if death_rate_in_interval.loc['90'] == 0:
- death_rate_in_interval.loc['90'] = death_rate_in_interval.loc['85-89']
-
- condition = number_of_deaths_by_sex > (
-
- person_years_by_sex / interval_width / interval_width)
-
- probability_of_dying_in_interval = pd.Series(index=number_of_deaths_by_sex.index, dtype=float)
-
- probability_of_dying_in_interval[condition] = 1
-
- probability_of_dying_in_interval[~condition] = interval_width * death_rate_in_interval / (
-
- 1 + interval_width * (1 - fraction_of_last_age_survived) * death_rate_in_interval)
-
- probability_of_dying_in_interval.at['90'] = 1
- return probability_of_dying_in_interval, death_rate_in_interval
-
def _estimate_life_expectancy(
_person_years_at_risk: pd.Series,
@@ -154,11 +124,29 @@ def _estimate_life_expectancy(
# separate male and female data
for sex in ['M', 'F']:
- probability_of_dying_in_interval, death_rate_in_interval = calculate_probability_of_dying(interval_width,
- fraction_of_last_age_survived,
- sex,
- _person_years_at_risk,
- _number_of_deaths_in_interval)
+ person_years_by_sex = _person_years_at_risk.xs(key=sex, level='sex')
+ number_of_deaths_by_sex = _number_of_deaths_in_interval.xs(key=sex, level='sex')
+
+ death_rate_in_interval = number_of_deaths_by_sex / person_years_by_sex
+ # if no deaths or person-years, produces nan
+ death_rate_in_interval = death_rate_in_interval.fillna(0)
+ # if no deaths in age 90+, set death rate equal to value in age 85-89
+ if death_rate_in_interval.loc['90'] == 0:
+ death_rate_in_interval.loc['90'] = death_rate_in_interval.loc['85-89']
+
+ # Calculate the probability of dying in the interval
+ # condition checks whether the observed number deaths is significantly higher than would be expected
+ # based on population years at risk and survival fraction
+ # if true, suggests very high mortality rates and returns value 1
+ condition = number_of_deaths_by_sex > (
+ person_years_by_sex / interval_width / fraction_of_last_age_survived)
+ probability_of_dying_in_interval = pd.Series(index=number_of_deaths_by_sex.index, dtype=float)
+ probability_of_dying_in_interval[condition] = 1
+ probability_of_dying_in_interval[~condition] = interval_width * death_rate_in_interval / (
+ 1 + interval_width * (1 - fraction_of_last_age_survived) * death_rate_in_interval)
+ # all those surviving to final interval die during this interval
+ probability_of_dying_in_interval.at['90'] = 1
+
# number_alive_at_start_of_interval
# keep dtype as float in case using aggregated outputs
# note range stops BEFORE the specified number
@@ -260,90 +248,3 @@ def get_life_expectancy_estimates(
else:
return summarize(results=output, only_mean=False, collapse_columns=False)
-
-
-def _calculate_probability_of_premature_death_for_single_run(
- age_before_which_death_is_defined_as_premature: int,
- person_years_at_risk: pd.Series,
- number_of_deaths_in_interval: pd.Series
-) -> Dict[str, float]:
- """
- For a single run, estimate the probability of dying before the defined premature age for males and females.
- Returns: Dict (keys by "M" and "F" for the sex, values the estimated probability of dying before the defined
- premature age).
- """
- probability_of_premature_death = dict()
-
- age_group_labels = person_years_at_risk.index.get_level_values('age_group').unique()
- interval_width = [
- 5 if '90' in interval else int(interval.split('-')[1]) - int(interval.split('-')[0]) + 1
- if '-' in interval else 1 for interval in age_group_labels.categories
- ]
- number_age_groups = len(interval_width)
- fraction_of_last_age_survived = pd.Series([0.5] * number_age_groups, index=age_group_labels)
-
- for sex in ['M', 'F']:
- probability_of_dying_in_interval, death_rate_in_interval = calculate_probability_of_dying(interval_width,
- fraction_of_last_age_survived,
- sex,
- person_years_at_risk,
- number_of_deaths_in_interval)
-
- # Calculate cumulative probability of dying before the defined premature age
- cumulative_probability_of_dying = 0
- proportion_alive_at_start_of_interval = 1.0
-
- for age_group, prob in probability_of_dying_in_interval.items():
- if int(age_group.split('-')[0]) >= age_before_which_death_is_defined_as_premature:
- break
- cumulative_probability_of_dying += proportion_alive_at_start_of_interval * prob
- proportion_alive_at_start_of_interval *= (1 - prob)
-
- probability_of_premature_death[sex] = cumulative_probability_of_dying
-
- return probability_of_premature_death
-
-
-def get_probability_of_premature_death(
- results_folder: Path,
- target_period: Tuple[datetime.date, datetime.date],
- summary: bool = True,
- age_before_which_death_is_defined_as_premature: int = 70
-) -> pd.DataFrame:
- """
- Produces sets of probability of premature death for each draw/run.
-
- Args:
- - results_folder (PosixPath): The path to the results folder containing log, `tlo.methods.demography`
- - target period (tuple of dates): Declare the date range (inclusively) in which the probability is to be estimated.
- - summary (bool): Declare whether to return a summarized value (mean with 95% uncertainty intervals)
- or return the estimate for each draw/run.
- - age_before_which_death_is_defined_as_premature (int): proposed in defined in Norheim et al.(2015) to be 70 years
-
- Returns:
- - pd.DataFrame: The DataFrame with the probability estimates for every draw/run in the results folder;
- or, with option `summary=True`, summarized (central, lower, upper estimates) for each draw.
- """
- info = get_scenario_info(results_folder)
- deaths = _num_deaths_by_age_group(results_folder, target_period)
- person_years = _aggregate_person_years_by_age(results_folder, target_period)
-
- prob_for_each_draw_and_run = dict()
-
- for draw in range(info['number_of_draws']):
- for run in range(info['runs_per_draw']):
- prob_for_each_draw_and_run[(draw, run)] = _calculate_probability_of_premature_death_for_single_run(
- age_before_which_death_is_defined_as_premature=age_before_which_death_is_defined_as_premature,
- number_of_deaths_in_interval=deaths[(draw, run)],
- person_years_at_risk=person_years[(draw, run)]
- )
-
- output = pd.DataFrame.from_dict(prob_for_each_draw_and_run)
- output.index.name = "sex"
- output.columns = output.columns.set_names(level=[0, 1], names=['draw', 'run'])
-
- if not summary:
- return output
-
- else:
- return summarize(results=output, only_mean=False, collapse_columns=False)
diff --git a/src/tlo/analysis/utils.py b/src/tlo/analysis/utils.py
index e605400332..344a094ef3 100644
--- a/src/tlo/analysis/utils.py
+++ b/src/tlo/analysis/utils.py
@@ -1,7 +1,6 @@
"""
General utility functions for TLO analysis
"""
-import fileinput
import gzip
import json
import os
@@ -87,40 +86,6 @@ def parse_log_file(log_filepath, level: int = logging.INFO):
return LogsDict({name: handle.name for name, handle in module_name_to_filehandle.items()}, level)
-def merge_log_files(log_path_1: Path, log_path_2: Path, output_path: Path) -> None:
- """Merge two log files, skipping any repeated header lines.
-
- :param log_path_1: Path to first log file to merge. Records from this log file will
- appear first in merged log file.
- :param log_path_2: Path to second log file to merge. Records from this log file will
- appear after those in log file at `log_path_1` and any header lines in this file
- which are also present in log file at `log_path_1` will be skipped.
- :param output_path: Path to write merged log file to. Must not be one of `log_path_1`
- or `log_path_2` as data is read from files while writing to this path.
- """
- if output_path == log_path_1 or output_path == log_path_2:
- msg = "output_path must not be equal to log_path_1 or log_path_2"
- raise ValueError(msg)
- with fileinput.input(files=(log_path_1, log_path_2), mode="r") as log_lines:
- with output_path.open("w") as output_file:
- written_header_lines = {}
- for log_line in log_lines:
- log_data = json.loads(log_line)
- if "type" in log_data and log_data["type"] == "header":
- if log_data["uuid"] in written_header_lines:
- previous_header_line = written_header_lines[log_data["uuid"]]
- if previous_header_line == log_line:
- continue
- else:
- msg = (
- "Inconsistent header lines with matching UUIDs found when merging logs:\n"
- f"{previous_header_line}\n{log_line}\n"
- )
- raise RuntimeError(msg)
- written_header_lines[log_data["uuid"]] = log_line
- output_file.write(log_line)
-
-
def write_log_to_excel(filename, log_dataframes):
"""Takes the output of parse_log_file() and creates an Excel file from dataframes"""
metadata = list()
@@ -325,9 +290,7 @@ def generate_series(dataframe: pd.DataFrame) -> pd.Series:
try:
df: pd.DataFrame = load_pickled_dataframes(results_folder, draw, run, module)[module][key]
output_from_eval: pd.Series = generate_series(df)
- assert isinstance(output_from_eval, pd.Series), (
- 'Custom command does not generate a pd.Series'
- )
+ assert pd.Series == type(output_from_eval), 'Custom command does not generate a pd.Series'
if do_scaling:
res[draw_run] = output_from_eval * get_multiplier(draw, run)
else:
@@ -1167,41 +1130,6 @@ def get_parameters_for_status_quo() -> Dict:
},
}
-def get_parameters_for_standard_mode2_runs() -> Dict:
- """
- Returns a dictionary of parameters and their updated values to indicate
- the "standard mode 2" scenario.
-
- The return dict is in the form:
- e.g. {
- 'Depression': {
- 'pr_assessed_for_depression_for_perinatal_female': 1.0,
- 'pr_assessed_for_depression_in_generic_appt_level1': 1.0,
- },
- 'Hiv': {
- 'prob_start_art_or_vs': 1.0,
- }
- }
- """
-
- return {
- "SymptomManager": {
- "spurious_symptoms": True,
- },
- "HealthSystem": {
- 'Service_Availability': ['*'],
- "use_funded_or_actual_staffing": "actual",
- "mode_appt_constraints": 1,
- "mode_appt_constraints_postSwitch": 2, # <-- Include a transition to mode 2, to pick up any issues with this
- "year_mode_switch": 2012, # <-- Could make this quite soon, but I'd say >1 year
- "tclose_overwrite": 1, # <-- In most of our runs in mode 2, we chose to overwrite tclose
- "tclose_days_offset_overwrite": 7, # <-- and usually set it to 7.
- "cons_availability": "default",
- "beds_availability": "default",
- "equip_availability": "all", # <--- NB. Existing calibration is assuming all equipment is available
- },
- }
-
def get_parameters_for_improved_healthsystem_and_healthcare_seeking(
resourcefilepath: Path,
diff --git a/src/tlo/bitset_handler/bitset_extension.py b/src/tlo/bitset_handler/bitset_extension.py
deleted file mode 100644
index 92d7af734f..0000000000
--- a/src/tlo/bitset_handler/bitset_extension.py
+++ /dev/null
@@ -1,706 +0,0 @@
-from __future__ import annotations
-
-import operator
-import re
-from typing import (
- TYPE_CHECKING,
- Any,
- Callable,
- Dict,
- Iterable,
- List,
- Optional,
- Sequence,
- Set,
- Tuple,
- Type,
- TypeAlias,
-)
-
-import numpy as np
-import pandas as pd
-from numpy.typing import NDArray
-from pandas._typing import TakeIndexer, type_t
-from pandas.core.arrays.base import ExtensionArray
-from pandas.core.dtypes.base import ExtensionDtype
-
-if TYPE_CHECKING:
- from pandas._typing import type_t
-
-BYTE_WIDTH = 8
-BooleanArray: TypeAlias = np.ndarray[bool]
-CastableForPandasOps: TypeAlias = (
- "ElementType"
- | Iterable["ElementType"]
- | NDArray[np.uint8]
- | NDArray[np.bytes_]
- | "BitsetArray"
-)
-SingletonForPandasOps: TypeAlias = "ElementType" | Iterable["ElementType"]
-# Assume nodes are strings, else we can't construct from string when passed the name!
-# We can likely get around this with some careful planning, but we'd have to figure out how
-# to pass type-metadata for the elements from inside the output of self.name, so that casting
-# was successful.
-ElementType: TypeAlias = str
-
-
-class BitsetDtype(ExtensionDtype):
- """
- A Bitset is represented by a fixed-width string, whose characters are each a uint8.
- Elements of the set map 1:1 to these characters.
-
- If the elements set is indexed starting from 0, then:
- - The quotient of these indices (modulo 8) is the character within the string that contains the bit representing the element,
- - The remainder (modulo 8) is the index within said character that represents the element itself.
-
- The element map takes an element of the bitset as a key, and returns a tuple whose first element is the
- corresponding string-character index, and the latter the uint8 representation of the element within that
- string character.
- """
- _element_map: Dict[ElementType, Tuple[int, np.uint8]]
- _elements: Tuple[ElementType]
- _index_map: Dict[Tuple[int, np.uint8], ElementType]
- _metadata = ("_elements",)
-
- @classmethod
- def construct_array_type(cls) -> type_t[BitsetArray]:
- return BitsetArray
-
- @classmethod
- def construct_from_string(cls, string: str) -> BitsetDtype:
- """
- Construct an instance of this class by passing in a string of the form
- that str() produces.
-
- That is, given a string of the form
- bitset(#elements): e1, e2, e3, ...
-
- this method will return a BitsetDtype with elements e1, e2, e3, ... etc.
-
- The bitset(#elements): prefix is not required, simply passing a comma-separated
- string of values will suffice to construct a bitset with those elements.
- The prefix is typically supplied when constructing an implicit instance as part of
- a call to `pd.Series` with the `dtype` parameter set to a string,
- """
- if not isinstance(string, str):
- raise TypeError(f"'construct_from_string' expects a string, got {type(string)}")
-
- string_has_bitset_prefix = re.match(r"bitset\((\d+)\):", string)
- n_elements = None
- if string_has_bitset_prefix:
- prefix = string_has_bitset_prefix.group(0)
- # Remove prefix
- string = string.removeprefix(prefix)
- # Extract number of elements if provided though
- n_elements = int(re.search(r"(\d+)", prefix).group(0))
- if "," not in string:
- raise TypeError(
- "Need at least 2 (comma-separated) elements in string to construct bitset."
- )
- else:
- iterable_values = tuple(s.strip() for s in string.split(","))
- if n_elements is not None and len(iterable_values) != n_elements:
- raise ValueError(
- f"Requested bitset with {n_elements} elements, but provided {len(iterable_values)} elements: {iterable_values}"
- )
- return BitsetDtype(s.strip() for s in string.split(","))
-
- @property
- def elements(self) -> Tuple[ElementType]:
- return self._elements
-
- @property
- def fixed_width(self) -> int:
- """
- Fixed-length of the character string that represents this bitset.
- """
- return (self.n_elements - 1) // BYTE_WIDTH + 1
-
- @property
- def n_elements(self) -> int:
- return len(self._elements)
-
- @property
- def na_value(self) -> np.bytes_:
- return self.type(self.fixed_width)
-
- @property
- def name(self) -> str:
- return self.__str__()
-
- @property
- def np_array_dtype(self) -> np.dtype:
- return np.dtype((bytes, self.fixed_width))
-
- @property
- def type(self) -> Type[np.bytes_]:
- return self.np_array_dtype.type
-
- def __init__(self, elements: Iterable[ElementType]) -> None:
- # Take only unique elements.
- # Sort elements alphabetically for consistency when constructing Bitsets that
- # represent the same items.
- # Cast all element types to strings so that construct_from_string does not need
- # metadata about the type of each element.
- provided_elements = sorted([e for e in elements])
- if not all(
- isinstance(e, ElementType) for e in provided_elements
- ):
- raise TypeError(f"BitSet elements must type {ElementType}")
- self._elements = tuple(
- sorted(set(provided_elements), key=lambda x: provided_elements.index(x))
- )
-
- if len(self._elements) <= 1:
- raise ValueError("Bitsets must have at least 2 possible elements (use bool for 1-element sets).")
-
- # Setup the element map and its inverse, one-time initialisation cost.
- self._element_map = {
- e: (index // BYTE_WIDTH, np.uint8(2 ** (index % BYTE_WIDTH)))
- for index, e in enumerate(self._elements)
- }
- self._index_map = {loc: element for element, loc in self._element_map.items()}
-
- def __repr__(self) -> str:
- return f"bitset({self.n_elements}): {', '.join(str(e) for e in self._elements)}"
-
- def __str__(self) -> str:
- return self.__repr__()
-
- def as_bytes(self, collection: Iterable[ElementType] | ElementType) -> np.bytes_:
- """
- Return the bytes representation of this set or single element.
- """
- return np.bytes_(self.as_uint8_array(collection))
-
- def as_set(self, binary_repr: np.bytes_) -> Set[ElementType]:
- """
- Return the set corresponding to the binary representation provided.
- """
- elements_in_set = set()
- for char_index, byte_value in enumerate(binary_repr):
- bin_rep = format(byte_value, "b")
- elements_in_set |= {
- self._index_map[(char_index, np.uint8(2**i))]
- for i, bit in enumerate(reversed(bin_rep))
- if bit == "1"
- }
- return elements_in_set
-
- def as_uint8_array(self, collection: Iterable[ElementType] | ElementType) -> NDArray[np.uint8]:
- """
- Return the collection of elements as a 1D array of ``self.fixed_width`` uint8s.
- Each uint8 corresponds to the bitwise representation of a single character
- in a character string.
-
- A single element will be broadcast to a (1,) numpy array.
- """
- if isinstance(collection, ElementType):
- collection = set(collection)
-
- output = np.zeros((self.fixed_width, 1), dtype=np.uint8)
- for element in collection:
- char, bin_repr = self._element_map[element]
- output[char] |= bin_repr
- return output.squeeze(axis=1)
-
- def element_loc(self, element: ElementType) -> Tuple[int, np.uint8]:
- """
- Location in of the bit corresponding to the element in this bitset.
-
- Each element in the bitset is mapped to a single bit via the _element_map, and
- can be located by specifying both:
- - The index of the character in the fixed-width string that represents the bitset.
- - The power of 2 within the uint8 representation of the the single character that corresponds to the element.
-
- For example, a bitset of 18 elements is stored as a fixed-width string of 3 characters,
- giving 24 bits to utilise. These are further subdivided into groups of 8, the first 8
- corresponding to the uint8 representation of the 0-indexed character, and so on. Each element within
- this bitset is assigned a power of two within one of the character representations.
-
- :param element: Element value to locate.
- :returns: The character index, and ``np.uint8`` representation of the element, unpacked in that order.
- """
- return self._element_map[element]
-
-
-class BitsetArray(ExtensionArray):
- """
- Represents a series of Bitsets; each element in the series is a fixed-width bytestring,
- which represents some possible combination of elements of a bitset as defined by
- ``self.dtype``.
-
- When extracting a single entry via ``.loc`` or ``.at``, the value returned is a ``set``.
- This means that operations such as ``self.loc[0] |= {"1"}`` will behave as set operations
- from base Python. This is achieved by setting the behaviour of the ``__setitem__`` method
- to interpret ``set`` values as representations of the underlying bitset, thus causing them
- to be cast to their bytestring representation being being assigned.
-
- Supported Operations (slices)
- -----------------------------
- When operating on slices or masks of the series, we have to re-implement the desired operators
- so that users can continue to pass ``set``s as scalar arguments on the left. As a general rule
- of thumb, if a binary operator can be performed on ``set``s, it will also work identically,
- but entry-wise, on a bitset series.
-
- ``NodeType`` instances will be cast to ``set``s if provided as singletons. Comparisons will be
- performed entry-wise if a suitable vector of values is provided as the comparison target.
-
- Currently implemented methods are:
-
- = :
- Directly assign the value on the right to the entry/entries on the left.
- +, | :
- Perform union of the values on the left with those on the right.
- +=, |= :
- In-place union; add values on the right to the sets on the left.
- & :
- Perform intersection of the values on the left with those on the right.
- &= :
- In-place intersection; retain only elements on the left that appear on the right.
- -, -= :
- Remove the values on the right from the sets on the left.
- <, <= :
- Entry-wise subset (strict subset) with the values on the right.
- >, >= :
- Entry-wise superset (strict superset) with the values on the right.
- Note that the >= operation is the equivalent of entry-wise "if the values on the right
- are contained in the bitsets on the left".
- """
-
- _data: NDArray[np.bytes_]
- _dtype: BitsetDtype
-
- @staticmethod
- def uint8s_to_byte_string(arr: np.ndarray[np.uint8]) -> NDArray[np.bytes_]:
- """
- Returns a view of an array of ``np.uint8``s of shape ``(M, N)``
- as an array of ``M`` fixed-width byte strings of size ``N``.
- """
- fixed_width = arr.shape[1]
- return arr.view(f"{fixed_width}S").squeeze()
-
- @classmethod
- def _concat_same_type(cls, to_concat: Sequence[BitsetArray]) -> BitsetArray:
- concat_data = np.concatenate(bsa._data for bsa in to_concat)
- return cls(concat_data, to_concat[0].dtype)
-
- @classmethod
- def _from_sequence(
- cls, scalars: Iterable[Set[ElementType] | ElementType], *, dtype: BitsetDtype | None = None, copy: bool = False
- ) -> BitsetArray:
- """
- Construct a new BitSetArray from a sequence of scalars.
-
- :param scalars: Sequence of sets of elements (or single-values to be interpreted as single-element sets).
- :param dtype: Cast to this datatype, only BitsetDtype is supported if not None.
- If None, an attempt will be made to construct an appropriate BitsetDtype using the scalar values provided.
- :param copy: If True, copy the underlying data. Default False.
- """
- # Check that we have only been passed sets as scalars. Implicitly convert single-items to sets.
- for i, s in enumerate(scalars):
- if not isinstance(s, set):
- if isinstance(s, ElementType):
- scalars[i] = set(s)
- else:
- raise ValueError(f"{s} cannot be cast to an element of a bitset.")
-
- # If no dtype has been provided, attempt to construct an appropriate BitsetDtype.
- if dtype is None:
- # Determine the elements in the bitset by looking through the scalars
- all_elements = set().union(scalars)
- dtype = BitsetDtype(all_elements)
- elif not isinstance(dtype, BitsetDtype):
- raise TypeError(f"BitsetArray cannot be constructed with dtype {dtype}")
-
- # With an appropriate dtype, we can construct the data array to pass to the constructor.
- # We will need to convert each of our scalars to their binary representations before passing though.
- data = np.zeros((len(scalars),), dtype=dtype.np_array_dtype)
- view_format = f"{dtype.fixed_width}B" if dtype.fixed_width != 1 else "(1,1)B"
- data_view = data.view(view_format)
- for series_index, s in enumerate(scalars):
- for element in s:
- char, u8_repr = dtype.element_loc(element=element)
- data_view[series_index, char] |= u8_repr
- return cls(data, dtype, copy=copy)
-
- @classmethod
- def _from_factorized(cls, uniques: np.ndarray, original: BitsetArray) -> BitsetArray:
- return cls(uniques, original.dtype)
-
- @property
- def _uint8_view_format(self) -> str:
- """
- Format string to be applied to self._data, so that the output of
-
- self._data.view()
-
- returns a numpy array of shape (len(self), self.dtype.fixed_width)
- and dtype uint8.
- """
- return f"({self.dtype.fixed_width},)B"
-
- @property
- def _uint8_view(self) -> NDArray[np.bytes_]:
- """
- Returns a view of the fixed-width byte strings stored in ``self._data``
- as an array of ``numpy.uint8``s, with shape
-
- ``(len(self._data), self.dtype.fixed_width)``.
-
- Each row ``i`` of this view corresponds to a bitset stored in this array.
- The value at index ``i, j`` in this view is the ``uint8`` that represents
- character ``j`` in ``self._data[i]``, which can have bitwise operations
- performed on it.
- """
- return self._data.view(self._uint8_view_format)
-
- @property
- def as_sets(self) -> List[Set[ElementType]]:
- """
- Return a list whose entry i is the set representation of the
- bitset in entry i of this array.
- """
- return [self.dtype.as_set(x) for x in self._data]
-
- @property
- def dtype(self) -> BitsetDtype:
- return self._dtype
-
- @property
- def nbytes(self) -> int:
- return self._data.nbytes
-
- def __init__(
- self,
- data: Iterable | np.ndarray,
- dtype: BitsetDtype,
- copy: bool = False,
- ) -> None:
- """ """
- if not isinstance(dtype, BitsetDtype):
- raise TypeError("BitsetArray must have BitsetDtype data.")
-
- self._data = np.array(data, copy=copy, dtype=dtype.type)
- self._dtype = dtype
-
- def __add__(
- self, other: CastableForPandasOps
- ) -> BitsetArray:
- """
- Entry-wise union with other.
-
- - If other is ``NodeType`` or ``Iterable[NodeType]``, perform entry-wise OR with the set
- representing the passed element values.
- - If other is ``BitsetArray`` of compatible shape, take entry-wise union.
- - If other is compatible ``np.ndarray``, take entry-wise union.
-
- Under the hood this is bitwise OR with other; self OR other.
- """
- return BitsetArray(
- self.__operate_bitwise(
- lambda A, B: A | B, other, return_as_bytestring=True
- ),
- dtype=self.dtype,
- )
-
- def __and__(self, other: CastableForPandasOps
- ) -> BitsetArray:
- """
- Entry-wise intersection with other.
-
- - If other is ``NodeType`` or ``Iterable[NodeType]``, perform entry-wise AND with the set
- representing the passed element values.
- - If other is ``BitsetArray`` of compatible shape, take entry-wise intersection.
- - If other is compatible ``np.ndarray``, take entry-wise intersection.
-
- Under the hood this is bitwise AND with other; self AND other.
- """
- return BitsetArray(
- self.__operate_bitwise(
- lambda A, B: A & B, other, return_as_bytestring=True
- ),
- dtype=self.dtype,
- )
-
- def __cast_before_comparison_op(
- self, value: CastableForPandasOps
- ) -> Set[ElementType] | bool:
- """
- Common steps taken before employing comparison operations on this class.
-
- Converts the value passed (as safely as possible) to a set, which can then
- be compared with the bitsets stored in the instance.
-
- Return values are the converted value, and whether this value should be considered
- a scalar-set (False) or a collection of sets (True).
- """
- if isinstance(value, ElementType):
- return set(value), False
- elif isinstance(value, set):
- return value, False
- elif isinstance(value, BitsetArray):
- return value.as_sets, True
- elif isinstance(value, np.ndarray):
- return [
- self.dtype.as_set(bytestr)
- for bytestr in self.uint8s_to_byte_string(self.__cast_to_uint8(value))
- ]
- # Last ditch attempt - we might have been given a list of sets, for example...
- try:
- value = set(value)
- if all([isinstance(item, ElementType) for item in value]):
- return value, False
- elif all([isinstance(item, set) for item in value]):
- return value, True
- except Exception as e:
- raise ValueError(f"Cannot compare bitsets with: {value}") from e
-
- def __cast_to_uint8(self, other: CastableForPandasOps) -> NDArray[np.uint8]:
- """
- Casts the passed object to a ``np.uint8`` array that is compatible with bitwise operations
- on ``self._uint8_view``. See the docstring for behaviour in the various usage cases.
-
- Scalar elements:
- Cast to single-element sets, then treated as set.
-
- Sets:
- Are converted to the (array of) uint8s that represents the set.
-
- ``np.ndarray``s of ``np.uint8``
- Are returned if they have the same number of columns as ``self._uint8_view``.
-
- ``np.ndarray``s of ``np.dtype("Sx")``
- If ``x`` corresponds to the same fixed-width as ``self.dtype.np_array_dtype``, are cast
- to the corresponding ``np.uint8`` view, like ``self._uint8_view`` is from ``self._data``.
-
- BitsetArrays
- Return their ``_uint8_view`` attribute.
- """
- if isinstance(other, ElementType):
- # Treat single-elements as single-element sets
- other = set(other)
- if isinstance(other, BitsetArray):
- if self.dtype != other.dtype:
- raise TypeError("Cannot cast a different Bitset to this one!")
- else:
- cast = other._uint8_view
- elif isinstance(other, np.ndarray):
- if other.size == 0:
- cast = self.dtype.as_uint8_array({})
- elif (other == other[0]).all():
- cast = self.dtype.as_uint8_array(other[0])
- elif other.dtype == np.uint8 and other.shape[0] == self._uint8_view.shape[0]:
- # Compatible uint8s, possibly a view of another fixed-width bytestring array
- cast = other
- elif other.dtype == self.dtype.np_array_dtype:
- # An array of compatible fixed-width bytestrings
- cast = other.view(self._uint8_view_format)
- elif other.dtype == object and all(isinstance(s, (ElementType, set)) for s in other):
- # We might have been passed an object array, where each object is a set or singleton that
- # we need to convert.
- as_bytes = np.array([self.dtype.as_bytes(s) for s in other], dtype=self.dtype.np_array_dtype)
- cast = as_bytes.view(self._uint8_view_format)
- else:
- raise ValueError(f"Cannot convert {other} to an array of uint8s representing a bitset")
- else:
- # Must be a collection of elements (or will error), so cast.
- cast = self.dtype.as_uint8_array(other)
- return cast
-
- def __comparison_op(self, other: CastableForPandasOps, op: Callable[[Set[ElementType], Set[ElementType]], bool]) -> BooleanArray:
- """
- Abstract method for strict and non-strict comparison operations.
-
- Notably, __eq__ does not redirect here since it is more efficient for us to convert
- the single value to a bytestring and use numpy array comparison.
-
- For the other set comparison methods however, it's easier as a first implementation
- for us to convert to sets and run the set operations. If there was a Pythonic way
- of doing "bitwise less than" and "bitwise greater than", we could instead take the
- same approach as in __operate_bitwise:
- - Convert the inputs to ``NDArray[np.bytes_]``.
- - Compare using __operate_bitwise with self._data.
-
- which would avoid us having to cast everything to a list and then do a list
- comprehension (the numpy direct array comparison should be faster).
- """
- if isinstance(other, (pd.Series, pd.DataFrame, pd.Index)):
- return NotImplemented
- other, is_vector = self.__cast_before_comparison_op(other)
-
- if is_vector:
- return np.array([op(s, other[i]) for i, s in enumerate(self.as_sets)])
- else:
- return np.array([op(s, other) for s in self.as_sets], dtype=bool)
-
- def __contains__(self, item: SingletonForPandasOps | Any) -> BooleanArray | bool:
- if isinstance(item, ElementType):
- item = set(item)
- if isinstance(item, set):
- return item in self.as_sets
- else:
- return super().__contains__(item)
-
- def __eq__(self, other) -> bool:
- if isinstance(other, (pd.Series, pd.DataFrame, pd.Index)):
- return NotImplemented
- elif isinstance(other, ElementType):
- other = set(other)
-
- if isinstance(other, set):
- ans = self._data == self.dtype.as_bytes(other)
- else:
- ans = self._data == other
- return np.squeeze(ans)
-
- def __getitem__(self, item: int | slice | NDArray) -> BitsetArray:
- return (
- self.dtype.as_set(self._data[item])
- if isinstance(item, int)
- else BitsetArray(self._data[item], dtype=self.dtype)
- )
-
- def __ge__(self, other: SingletonForPandasOps) -> BooleanArray:
- """
- Entry-wise non-strict superset: self >= other_set.
- """
- return self.__comparison_op(other, operator.ge)
-
- def __gt__(self, other: SingletonForPandasOps) -> BooleanArray:
- """
- Entry-wise strict superset: self > other_set.
- """
- return self.__comparison_op(other, operator.gt)
-
- def __len__(self) -> int:
- return self._data.shape[0]
-
- def __le__(self, other: SingletonForPandasOps) -> BooleanArray:
- """
- Entry-wise non-strict subset: self <= other_set.
- """
- return self.__comparison_op(other, operator.le)
-
- def __lt__(self, other: SingletonForPandasOps) -> BooleanArray:
- """
- Entry-wise strict subset: self < other_set.
- """
- return self.__comparison_op(other, operator.lt)
-
- def __operate_bitwise(
- self,
- op: Callable[[NDArray[np.uint8], NDArray[np.uint8]], NDArray[np.uint8]],
- r_value: CastableForPandasOps,
- l_value: Optional[CastableForPandasOps] = None,
- return_as_bytestring: bool = False,
- ) -> NDArray[np.bytes_] | NDArray[np.uint8]:
- """
- Perform a bitwise operation on two compatible ``np.ndarray``s of ``np.uint8``s.
-
- By default, the left value passed to the operator is assumed to be ``self._uint8_data``.
-
- Return value is the result of the bitwise operation, as an array of uint8s. If you wish
- to have this converted to the corresponding bytestring(s) before returning, use the
- return_as_bytestring argument.
-
- :param op: Bitwise operation to perform on input values.
- :param r_value: Right-value to pass to the operator.
- :param l_value: Left-value to pass to the operator.
- :param return_as_bytestring: Result will be returned as a fixed-width bytestring.
- """
- l_value = self._uint8_view if l_value is None else self.__cast_to_uint8(l_value)
- op_result = op(l_value, self.__cast_to_uint8(r_value))
- if return_as_bytestring:
- op_result = self.uint8s_to_byte_string(op_result)
- return op_result
-
- def __or__(
- self, other: CastableForPandasOps
- ) -> BitsetArray:
- """
- Entry-wise union with other, delegating to ``self.__add__``.
-
- np.ndarrays of objects will attempt to interpret their elements as bitsets.
- """
- return self.__add__(other)
-
- def __setitem__(
- self,
- key: int | slice | NDArray,
- value: (
- np.bytes_
- | ElementType
- | Set[ElementType]
- | Sequence[np.bytes_ | ElementType| Set[ElementType]]
- ),
- ) -> None:
- if isinstance(value, ElementType) or isinstance(value, set):
- # Interpret this as a "scalar" set that we want to set all values to
- value = self.dtype.as_bytes(value)
- elif isinstance(value, np.bytes_):
- # Value is a scalar that we don't need to convert
- pass
- else:
- # Assume value is a sequence, and we will have to convert each value in turn
- value = [
- v if isinstance(v, np.bytes_) else self.dtype.as_bytes(v) for v in value
- ]
- self._data[key] = value
-
- def __sub__(
- self, other: CastableForPandasOps
- ) -> BitsetArray:
- """
- Remove elements from the Bitsets represented here.
-
- - If other is ``NodeType``, remove the single element from all series entries.
- - If other is ``Iterable[NodeType]``, remove all elements from all series entries.
- - If other is ``BitsetArray`` of compatible shape, take element-wise complements of series entries.
- - If other is compatible ``np.ndarray``, take element-wise complements of series entries.
-
- Under the hood this the bitwise operation self AND (NOT other).
- """
- return BitsetArray(
- self.__operate_bitwise(
- lambda A, B: A & (~B), other, return_as_bytestring=True
- ),
- dtype=self.dtype,
- )
-
- def _formatter(self, boxed: bool = False) -> Callable[[np.bytes_], str | None]:
- if boxed: # If rendering an individual data value
- return lambda x: ",".join(x) if x else "{}"
- return repr # Render the table itself
-
- def copy(self) -> BitsetArray:
- return BitsetArray(self._data, self.dtype, copy=True)
-
- def isna(self) -> NDArray:
- """
- TODO: This isn't a great way to express missing data, but equally a bitset doesn't really ever contain missing data...
- """
- return np.isnan(self._data)
-
- def take(
- self,
- indices: TakeIndexer,
- *,
- allow_fill: bool = False,
- fill_value: Optional[np.bytes_ | Set[ElementType]] = None,
- ) -> BitsetArray:
- if allow_fill:
- if isinstance(fill_value, set):
- fill_value = self.dtype.as_bytes(fill_value)
- elif fill_value is None:
- fill_value = self.dtype.na_value
- elif not isinstance(fill_value, self.dtype.type):
- raise TypeError(
- f"Fill value must be of type {self.dtype.type} (got {type(fill_value).__name__})"
- )
- scalars = np.empty((len(indices), ), dtype=self.dtype.type)
- scalars[indices[indices >= 0]] = self._data[indices[indices >= 0]]
- scalars[indices[indices < 0]] = fill_value
- else:
- scalars = np.take(self._data, indices)
- return self._from_sequence(scalars)
diff --git a/src/tlo/cli.py b/src/tlo/cli.py
index 578c3a6619..9e780911f9 100644
--- a/src/tlo/cli.py
+++ b/src/tlo/cli.py
@@ -13,9 +13,9 @@
import dateutil.parser
import pandas as pd
from azure import batch
+from azure.batch import batch_auth
from azure.batch import models as batch_models
from azure.batch.models import BatchErrorException
-from azure.common.credentials import ServicePrincipalCredentials
from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError
from azure.identity import DefaultAzureCredential
from azure.keyvault.secrets import SecretClient
@@ -44,23 +44,17 @@ def cli(ctx, config_file, verbose):
ctx.obj["verbose"] = verbose
-@cli.command(context_settings=dict(ignore_unknown_options=True))
+@cli.command()
@click.argument("scenario_file", type=click.Path(exists=True))
@click.option("--draw-only", is_flag=True, help="Only generate draws; do not run the simulation")
@click.option("--draw", "-d", nargs=2, type=int)
@click.option("--output-dir", type=str)
-@click.argument('scenario_args', nargs=-1, type=click.UNPROCESSED)
-def scenario_run(scenario_file, draw_only, draw: tuple, output_dir=None, scenario_args=None):
+def scenario_run(scenario_file, draw_only, draw: tuple, output_dir=None):
"""Run the specified scenario locally.
SCENARIO_FILE is path to file containing a scenario class
"""
scenario = load_scenario(scenario_file)
-
- # if we have other scenario arguments, parse them
- if scenario_args is not None:
- scenario.parse_arguments(scenario_args)
-
config = scenario.save_draws(return_config=True)
json_string = json.dumps(config, indent=2)
@@ -132,7 +126,9 @@ def batch_submit(ctx, scenario_file, asserts_on, more_memory, keep_pool_alive, i
azure_directory = f"{config['DEFAULT']['USERNAME']}/{job_id}"
batch_client = get_batch_client(
- config["BATCH"]["CLIENT_ID"], config["BATCH"]["SECRET"], config["AZURE"]["TENANT_ID"], config["BATCH"]["URL"]
+ config["BATCH"]["NAME"],
+ config["BATCH"]["KEY"],
+ config["BATCH"]["URL"]
)
create_file_share(
@@ -245,16 +241,8 @@ def batch_submit(ctx, scenario_file, asserts_on, more_memory, keep_pool_alive, i
try:
# Create the job that will run the tasks.
- create_job(
- batch_client,
- vm_size,
- pool_node_count,
- job_id,
- container_conf,
- [mount_configuration],
- keep_pool_alive,
- config["BATCH"]["SUBNET_ID"],
- )
+ create_job(batch_client, vm_size, pool_node_count, job_id,
+ container_conf, [mount_configuration], keep_pool_alive)
# Add the tasks to the job.
add_tasks(batch_client, user_identity, job_id, image_name,
@@ -303,7 +291,9 @@ def batch_terminate(ctx, job_id):
return
batch_client = get_batch_client(
- config["BATCH"]["CLIENT_ID"], config["BATCH"]["SECRET"], config["AZURE"]["TENANT_ID"], config["BATCH"]["URL"]
+ config["BATCH"]["NAME"],
+ config["BATCH"]["KEY"],
+ config["BATCH"]["URL"]
)
# check the job is running
@@ -335,9 +325,10 @@ def batch_job(ctx, job_id, raw, show_tasks):
print(">Querying batch system\r", end="")
config = load_config(ctx.obj['config_file'])
batch_client = get_batch_client(
- config["BATCH"]["CLIENT_ID"], config["BATCH"]["SECRET"], config["AZURE"]["TENANT_ID"], config["BATCH"]["URL"]
+ config["BATCH"]["NAME"],
+ config["BATCH"]["KEY"],
+ config["BATCH"]["URL"]
)
-
tasks = None
try:
@@ -405,7 +396,9 @@ def batch_list(ctx, status, n, find, username):
username = config["DEFAULT"]["USERNAME"]
batch_client = get_batch_client(
- config["BATCH"]["CLIENT_ID"], config["BATCH"]["SECRET"], config["AZURE"]["TENANT_ID"], config["BATCH"]["URL"]
+ config["BATCH"]["NAME"],
+ config["BATCH"]["KEY"],
+ config["BATCH"]["URL"]
)
# create client to connect to file share
@@ -582,12 +575,9 @@ def load_server_config(kv_uri, tenant_id) -> Dict[str, Dict]:
return {"STORAGE": storage_config, "BATCH": batch_config, "REGISTRY": registry_config}
-def get_batch_client(client_id, secret, tenant_id, url):
+def get_batch_client(name, key, url):
"""Create a Batch service client"""
- resource = "https://batch.core.windows.net/"
-
- credentials = ServicePrincipalCredentials(client_id=client_id, secret=secret, tenant=tenant_id, resource=resource)
-
+ credentials = batch_auth.SharedKeyCredentials(name, key)
batch_client = batch.BatchServiceClient(credentials, batch_url=url)
return batch_client
@@ -701,19 +691,10 @@ def upload_local_file(connection_string, local_file_path, share_name, dest_file_
print("ResourceNotFoundError:", ex.message)
-def create_job(
- batch_service_client,
- vm_size,
- pool_node_count,
- job_id,
- container_conf,
- mount_configuration,
- keep_pool_alive,
- subnet_id,
-):
+def create_job(batch_service_client, vm_size, pool_node_count, job_id,
+ container_conf, mount_configuration, keep_pool_alive):
"""Creates a job with the specified ID, associated with the specified pool.
- :param subnet_id:
:param batch_service_client: A Batch service client.
:type batch_service_client: `azure.batch.BatchServiceClient`
:param str vm_size: Type of virtual machine to use as pool.
@@ -753,11 +734,6 @@ def create_job(
$NodeDeallocationOption = taskcompletion;
"""
- network_configuration = batch_models.NetworkConfiguration(
- subnet_id=subnet_id,
- public_ip_address_configuration=batch_models.PublicIPAddressConfiguration(provision="noPublicIPAddresses"),
- )
-
pool = batch_models.PoolSpecification(
virtual_machine_configuration=virtual_machine_configuration,
vm_size=vm_size,
@@ -765,8 +741,6 @@ def create_job(
task_slots_per_node=1,
enable_auto_scale=True,
auto_scale_formula=auto_scale_formula,
- network_configuration=network_configuration,
- target_node_communication_mode="simplified",
)
auto_pool_specification = batch_models.AutoPoolSpecification(
diff --git a/src/tlo/core.py b/src/tlo/core.py
index 9fbbf08893..98553c4039 100644
--- a/src/tlo/core.py
+++ b/src/tlo/core.py
@@ -8,20 +8,28 @@
import json
from enum import Enum, auto
-from typing import TYPE_CHECKING, Any, Dict, FrozenSet, List, Optional
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, TypeAlias, Union
import numpy as np
import pandas as pd
if TYPE_CHECKING:
- from pathlib import Path
- from typing import Optional
+ from numpy.random import RandomState
- from tlo.methods import Metadata
- from tlo.methods.causes import Cause
- from tlo.population import Population
+ from tlo.methods.healthsystem import HealthSystem
+ from tlo.population import PatientDetails
from tlo.simulation import Simulation
+DiagnosisFunction: TypeAlias = Callable[[str, bool, bool], Any]
+ConsumablesChecker: TypeAlias = Callable[
+ [
+ Union[None, np.integer, int, List, Set, Dict],
+ Union[None, np.integer, int, List, Set, Dict],
+ ],
+ Union[bool, Dict],
+]
+IndividualPropertyUpdates: TypeAlias = Dict[str, Any]
+
class Types(Enum):
"""Possible types for parameters and properties.
@@ -80,7 +88,7 @@ class Specifiable:
Types.BITSET: int,
}
- def __init__(self, type_: Types, description: str, categories: List[str] = None):
+ def __init__(self, type_, description, categories=None):
"""Create a new Specifiable.
:param type_: an instance of Types giving the type of allowed values
@@ -98,16 +106,16 @@ def __init__(self, type_: Types, description: str, categories: List[str] = None)
self.categories = categories
@property
- def python_type(self) -> type:
+ def python_type(self):
"""Return the Python type corresponding to this Specifiable."""
return self.PYTHON_TYPE_MAP[self.type_]
@property
- def pandas_type(self) -> type:
+ def pandas_type(self):
"""Return the Pandas type corresponding to this Specifiable."""
return self.PANDAS_TYPE_MAP[self.type_]
- def __repr__(self) -> str:
+ def __repr__(self):
"""Return detailed description of Specifiable."""
delimiter = " === "
@@ -135,17 +143,8 @@ class Property(Specifiable):
object: float("nan"),
np.uint32: 0,
}
- _default_value_override: Any
- def __init__(
- self,
- type_: Types,
- description: str,
- categories: List[str] = None,
- *,
- ordered: bool = False,
- default_value: Optional[Any] = None,
- ) -> None:
+ def __init__(self, type_, description, categories=None, *, ordered=False):
"""Create a new property specification.
:param type_: An instance of ``Types`` giving the type of allowed values of this
@@ -155,53 +154,17 @@ def __init__(
``Types.CATEGORICAL``.
:param ordered: Whether categories are ordered if ``type_`` is
``Types.CATEGORICAL``.
- :param default_value: The default value for the property.
"""
if type_ in [Types.SERIES, Types.DATA_FRAME]:
raise TypeError("Property cannot be of type SERIES or DATA_FRAME.")
-
super().__init__(type_, description, categories)
self.ordered = ordered
- # Use _default_value setter method to set property initial value
- self._default_value = default_value
@property
- def _default_value(self) -> Any:
- """
- Default value for this property, which will be used to fill the respective columns
- of the population dataframe, for example.
-
- If not explicitly set, it will fall back on the ``PANDAS_TYPE_DEFAULT_TYPE_MAP``.
- If a value is provided, it must:
-
- - Be of the corresponding TYPE for the property.
- - If ``type_`` is ``Types.CATEGORICAL``, it must also be a possible category.
- """
- return (
- self.PANDAS_TYPE_DEFAULT_VALUE_MAP[self.pandas_type]
- if self._default_value_override is None
- else self._default_value_override
- )
+ def _default_value(self):
+ return self.PANDAS_TYPE_DEFAULT_VALUE_MAP[self.pandas_type]
- @_default_value.setter
- def _default_value(self, new_val: Any) -> None:
- if new_val is not None:
- # Check for valid category
- if self.type_ is Types.CATEGORICAL:
- if new_val not in self.categories:
- raise ValueError(
- f"Value {new_val} is not a valid category, so cannot be set as the default."
- )
- # If not categorical, check for valid data type for default
- elif not isinstance(new_val, self.python_type):
- raise ValueError(
- f"Trying to set a default value of type {type(new_val).__name__}, "
- f"which is different from Property's type of {type(self.python_type).__name__}."
- )
- # Outside block so that providing new_val = None reverts to Property-wide default.
- self._default_value_override = new_val
-
- def create_series(self, name: str, size: int) -> pd.Series:
+ def create_series(self, name, size):
"""Create a Pandas Series for this property.
The values will be left uninitialised.
@@ -250,47 +213,51 @@ class attribute on a subclass.
# Subclasses can override this to declare the set of initialisation dependencies
# Declares modules that need to be registered in simulation and initialised before
# this module
- INIT_DEPENDENCIES: FrozenSet[str] = frozenset()
+ INIT_DEPENDENCIES = frozenset()
# Subclasses can override this to declare the set of optional init. dependencies
# Declares modules that need to be registered in simulation and initialised before
# this module if they are present, but are not required otherwise
- OPTIONAL_INIT_DEPENDENCIES: FrozenSet[str] = frozenset()
+ OPTIONAL_INIT_DEPENDENCIES = frozenset()
# Subclasses can override this to declare the set of additional dependencies
# Declares any modules that need to be registered in simulation in addition to those
# in INIT_DEPENDENCIES to allow running simulation
- ADDITIONAL_DEPENDENCIES: FrozenSet[str] = frozenset()
+ ADDITIONAL_DEPENDENCIES = frozenset()
# Subclasses can override this to declare the set of modules that this module can be
# used in place of as a dependency
- ALTERNATIVE_TO: FrozenSet[str] = frozenset()
+ ALTERNATIVE_TO = frozenset()
# Subclasses can override this set to add metadata tags to their class
# See tlo.methods.Metadata class
- METADATA: FrozenSet[Metadata] = frozenset()
+ METADATA = {}
- # Subclasses can override this dict to declare the causes death that this module contributes to
+ # Subclasses can override this set to declare the causes death that this module contributes to
# This is a dict of the form { None:
+ @property
+ def healthsystem(self) -> HealthSystem:
+ return self.sim.modules["HealthSystem"]
+
+ def __init__(self, name=None):
"""Construct a new disease module ready to be included in a simulation.
Initialises an empty parameters dictionary and module-specific random number
@@ -303,7 +270,7 @@ def __init__(self, name: Optional[str] = None) -> None:
self.name = name or self.__class__.__name__
self.sim: Optional[Simulation] = None
- def load_parameters_from_dataframe(self, resource: pd.DataFrame) -> None:
+ def load_parameters_from_dataframe(self, resource: pd.DataFrame):
"""Automatically load parameters from resource dataframe, updating the class parameter dictionary
Goes through parameters dict self.PARAMETERS and updates the self.parameters with values
@@ -336,7 +303,7 @@ def load_parameters_from_dataframe(self, resource: pd.DataFrame) -> None:
f"The value of '{parameter_value}' for parameter '{parameter_name}' "
f"could not be parsed as a {parameter_definition.type_.name} data type"
)
- if parameter_definition.python_type is list:
+ if parameter_definition.python_type == list:
try:
# chose json.loads instead of save_eval
# because it raises error instead of joining two strings without a comma
@@ -364,7 +331,7 @@ def load_parameters_from_dataframe(self, resource: pd.DataFrame) -> None:
# Save the values to the parameters
self.parameters[parameter_name] = parameter_value
- def read_parameters(self, data_folder: str | Path) -> None:
+ def read_parameters(self, data_folder):
"""Read parameter values from file, if required.
Must be implemented by subclasses.
@@ -374,41 +341,23 @@ def read_parameters(self, data_folder: str | Path) -> None:
"""
raise NotImplementedError
- def initialise_population(self, population: Population) -> None:
+ def initialise_population(self, population):
"""Set our property values for the initial population.
+ Must be implemented by subclasses.
+
This method is called by the simulation when creating the initial population, and is
responsible for assigning initial values, for every individual, of those properties
'owned' by this module, i.e. those declared in its PROPERTIES dictionary.
- By default, all ``Property``s in ``self.PROPERTIES`` will have
- their columns in the population dataframe set to the default value.
-
- Modules that wish to implement this behaviour do not need to implement this method,
- it will be inherited automatically. Modules that wish to perform additional steps
- during the initialise_population stage should reimplement this method and call
-
- ```python
- super().initialise_population(population=population)
- ```
-
- at the beginning of the method, then proceed with their additional steps. Modules that
- do not wish to inherit this default behaviour should re-implement initialise_population
- without the call to ``super()`` above.
-
TODO: We probably need to declare somehow which properties we 'read' here, so the
simulation knows what order to initialise modules in!
- :param population: The population of individuals in the simulation.
+ :param population: the population of individuals
"""
- df = population.props
-
- for property_name, property in self.PROPERTIES.items():
- df.loc[df.is_alive, property_name] = (
- property._default_value
- )
+ raise NotImplementedError
- def initialise_simulation(self, sim: Simulation) -> None:
+ def initialise_simulation(self, sim):
"""Get ready for simulation start.
Must be implemented by subclasses.
@@ -419,7 +368,7 @@ def initialise_simulation(self, sim: Simulation) -> None:
"""
raise NotImplementedError
- def pre_initialise_population(self) -> None:
+ def pre_initialise_population(self):
"""Carry out any work before any populations have been initialised
This optional method allows access to all other registered modules, before any of
@@ -427,7 +376,7 @@ def pre_initialise_population(self) -> None:
when a module's properties rely upon information from other modules.
"""
- def on_birth(self, mother_id: int, child_id: int) -> None:
+ def on_birth(self, mother_id, child_id):
"""Initialise our properties for a newborn individual.
Must be implemented by subclasses.
@@ -439,6 +388,85 @@ def on_birth(self, mother_id: int, child_id: int) -> None:
"""
raise NotImplementedError
- def on_simulation_end(self) -> None:
+ def on_simulation_end(self):
"""This is called after the simulation has ended.
Modules do not need to declare this."""
+
+ def do_at_generic_first_appt(
+ self,
+ patient_id: int,
+ patient_details: Optional[PatientDetails],
+ symptoms: Optional[List[str]],
+ diagnosis_function: Optional[DiagnosisFunction],
+ consumables_checker: Optional[ConsumablesChecker],
+ facility_level: Optional[str],
+ treatment_id: Optional[str],
+ random_state: Optional[RandomState],
+ ) -> Union[IndividualPropertyUpdates, None]:
+ """
+ Actions to be take during a NON-emergency generic HSI.
+
+ Derived classes should overwrite this method so that they are
+ compatible with the HealthSystem module, and can schedule HSI
+ events when a patient presents symptoms indicative of the
+ corresponding illness or condition.
+
+ When overwriting, arguments that are not required can be left out
+ of the definition.
+ If done so, the method MUST take a **kwargs input to avoid errors
+ when looping over all disease modules and running their generic
+ HSI methods.
+
+ HSI_Events should be scheduled by the Module implementing this
+ method using the :py:meth:`Module.healthsystem.schedule_hsi` method.
+ However, they should not write updates back to the population
+ DataFrame in this method - these values should be returned as a
+ dictionary as described below:
+
+ The return value of this function should be a dictionary
+ containing any changes that need to be made to the individual's
+ row in the population DataFrame.
+ Key/value pairs should be the column name and the new value to
+ assign to the patient.
+ In the event no updates are required; return an object that evaluates
+ to False when cast to a bool. Your options are:
+ - Omit a return statement and value (preferred).
+ - Return an empty dictionary. Use this case when patient details
+ might need updating conditionally, on EG patient symptoms or consumable
+ availability. In which case, an empty dictionary should be created and
+ key-value pairs added to this dictionary as such conditionals are checked.
+ If no conditionals are met, the empty dictionary will be returned.
+ - Use a return statement with no values (use if the logic of your
+ module-specific method necessitates the explicit return).
+ - Return None (not recommended, use "return" on its own, as above).
+
+ :param patient_id: Row index (ID) of the individual target of the HSI event in the population DataFrame.
+ :param patient_details: Patient details as provided in the population DataFrame.
+ :param symptoms: List of symptoms the patient is experiencing.
+ :param diagnosis_function: A function that can run diagnosis tests based on the patient's symptoms.
+ :param consumables_checker: A function that can query the HealthSystem to check for available consumables.
+ :param facility_level: The level of the facility that the patient presented at.
+ :param treatment_id: The treatment id of the HSI event triggering the generic appointment.
+ :param random_state: Random number generator to be used when making random choices during event creation.
+ """
+
+ def do_at_generic_first_appt_emergency(
+ self,
+ patient_id: int,
+ patient_details: Optional[PatientDetails] = None,
+ symptoms: Optional[List[str]] = None,
+ diagnosis_function: Optional[DiagnosisFunction] = None,
+ consumables_checker: Optional[ConsumablesChecker] = None,
+ facility_level: Optional[str] = None,
+ treatment_id: Optional[str] = None,
+ random_state: Optional[RandomState] = None,
+ ) -> Union[IndividualPropertyUpdates, None]:
+ """
+ Actions to be take during an EMERGENCY generic HSI.
+ Call signature and return values are identical to the
+ :py:meth:`~Module.do_at_generic_first_appt` method.
+ Derived classes should overwrite this method so that they are
+ compatible with the HealthSystem module, and can schedule HSI
+ events when a patient presents symptoms indicative of the
+ corresponding illness or condition.
+ """
diff --git a/src/tlo/dependencies.py b/src/tlo/dependencies.py
index 03a847d315..8003b44328 100644
--- a/src/tlo/dependencies.py
+++ b/src/tlo/dependencies.py
@@ -57,67 +57,6 @@ def get_all_dependencies(
)
-def get_missing_dependencies(
- module_instances: Iterable[Module],
- get_dependencies: DependencyGetter = get_all_dependencies,
-) -> Set[str]:
- """Get the set of missing required dependencies if any from an iterable of modules.
-
- :param module_instances: Iterable of ``Module`` subclass instances to get missing
- dependencies for.
- :param get_dependencies: Callable which extracts the set of dependencies to check
- for from a module instance. Defaults to extracting all dependencies.
- :return: Set of ``Module`` subclass names corresponding to missing dependencies.
- """
- module_instances = list(module_instances)
- modules_present = {type(module).__name__ for module in module_instances}
- modules_present_are_alternatives_to = set.union(
- # Force conversion to set to avoid errors when using set.union with frozenset
- *(set(module.ALTERNATIVE_TO) for module in module_instances)
- )
- modules_required = set.union(
- *(set(get_dependencies(module, modules_present)) for module in module_instances)
- )
-
- missing_dependencies = modules_required - modules_present
- return (
- missing_dependencies - modules_present_are_alternatives_to
- )
-
-
-def initialise_missing_dependencies(modules: Iterable[Module], **module_kwargs) -> Set[Module]:
- """Get list of initialised instances of any missing dependencies for an iterable of modules.
-
- :param modules: Iterable of ``Module`` subclass instances to get instances of missing
- dependencies for.
- :param module_kwargs: Any keyword arguments to use when initialising missing
- module dependencies.
- :return: Set of ``Module`` subclass instances corresponding to missing dependencies.
- """
- module_class_map: Mapping[str, Type[Module]] = get_module_class_map(set())
- all_module_instances: list[Module] = list(modules)
-
- def add_missing_module_instances(modules: list[Module], all_missing_module_names: set[str]) -> None:
- """ add missing module instances to all_module_instances list
- :param modules: Iterable of registered modules
- :param all_missing_module_names: Set of missing module names
- """
- missing_dependencies: set[str] = get_missing_dependencies(
- modules, get_all_dependencies
- )
- if len(missing_dependencies) > 0:
- all_missing_module_names |= missing_dependencies
- missing_module_instances: list[Module] = [
- module_class_map[dependency](**module_kwargs)
- for dependency in missing_dependencies
- ]
- modules.extend(missing_module_instances)
- add_missing_module_instances(modules, all_missing_module_names)
-
- add_missing_module_instances(all_module_instances, set())
- return set(all_module_instances) - set(modules)
-
-
def get_all_required_dependencies(
module: Union[Module, Type[Module]],
module_names_present: Optional[Set[str]] = None
@@ -137,7 +76,7 @@ def get_all_required_dependencies(
def topologically_sort_modules(
module_instances: Iterable[Module],
- get_dependencies: DependencyGetter = get_init_dependencies
+ get_dependencies: DependencyGetter = get_init_dependencies,
) -> Generator[Module, None, None]:
"""Generator which yields topological sort of modules based on their dependencies.
@@ -181,7 +120,6 @@ def depth_first_search(module):
dependencies = get_dependencies(
module_instance_map[module], module_instance_map.keys()
)
-
for dependency in sorted(dependencies):
if dependency not in module_instance_map:
alternatives_with_instances = [
@@ -326,12 +264,23 @@ def check_dependencies_present(
:raises ModuleDependencyError: Raised if any dependencies are missing.
"""
- missing_dependencies = get_missing_dependencies(
- module_instances, get_dependencies
+ module_instances = list(module_instances)
+ modules_present = {type(module).__name__ for module in module_instances}
+ modules_present_are_alternatives_to = set.union(
+ # Force conversion to set to avoid errors when using set.union with frozenset
+ *(set(module.ALTERNATIVE_TO) for module in module_instances)
+ )
+ modules_required = set.union(
+ *(set(get_dependencies(module, modules_present)) for module in module_instances)
)
- if len(missing_dependencies) > 0:
+ missing_dependencies = modules_required - modules_present
+ missing_dependencies_without_alternatives_present = (
+ missing_dependencies - modules_present_are_alternatives_to
+ )
+ if not missing_dependencies_without_alternatives_present == set():
+
raise ModuleDependencyError(
'One or more required dependency is missing from the module list and no '
'alternative to this / these modules are available either: '
- f'{missing_dependencies}'
+ f'{missing_dependencies_without_alternatives_present}'
)
diff --git a/src/tlo/logging/__init__.py b/src/tlo/logging/__init__.py
index 7f1447f037..e17e5c37b5 100644
--- a/src/tlo/logging/__init__.py
+++ b/src/tlo/logging/__init__.py
@@ -1,27 +1,7 @@
-from .core import (
- CRITICAL,
- DEBUG,
- FATAL,
- INFO,
- WARNING,
- disable,
- getLogger,
- initialise,
- reset,
- set_output_file,
-)
-from .helpers import set_logging_levels
+from .core import CRITICAL, DEBUG, FATAL, INFO, WARNING, disable, getLogger
+from .helpers import init_logging, set_logging_levels, set_output_file, set_simulation
-__all__ = [
- "CRITICAL",
- "DEBUG",
- "FATAL",
- "INFO",
- "WARNING",
- "disable",
- "getLogger",
- "initialise",
- "reset",
- "set_output_file",
- "set_logging_levels",
-]
+__all__ = ['CRITICAL', 'DEBUG', 'FATAL', 'INFO', 'WARNING', 'disable', 'getLogger',
+ 'set_output_file', 'init_logging', 'set_simulation', 'set_logging_levels']
+
+init_logging()
diff --git a/src/tlo/logging/core.py b/src/tlo/logging/core.py
index dc3beaf2f1..e870e1f179 100644
--- a/src/tlo/logging/core.py
+++ b/src/tlo/logging/core.py
@@ -1,361 +1,217 @@
-from __future__ import annotations
-
import hashlib
import json
import logging as _logging
-import sys
-import warnings
-from functools import partialmethod
-from pathlib import Path
-from typing import Any, Callable, List, Optional, TypeAlias, Union
+from typing import Union
-import numpy as np
import pandas as pd
from tlo.logging import encoding
-LogLevel: TypeAlias = int
-LogData: TypeAlias = Union[str, dict, list, set, tuple, pd.DataFrame, pd.Series]
-SimulationDateGetter: TypeAlias = Callable[[], str]
-
-CRITICAL = _logging.CRITICAL
-DEBUG = _logging.DEBUG
-FATAL = _logging.FATAL
-INFO = _logging.INFO
-WARNING = _logging.WARNING
-_DEFAULT_LEVEL = INFO
-
-_DEFAULT_FORMATTER = _logging.Formatter("%(message)s")
-
-
-class InconsistentLoggedColumnsWarning(UserWarning):
- """Warning raised when structured log entry has different columns from header."""
-
-
-def _mock_simulation_date_getter() -> str:
- return "0000-00-00T00:00:00"
-
-
-_get_simulation_date: SimulationDateGetter = _mock_simulation_date_getter
-_loggers: dict[str, Logger] = {}
-
-
-def initialise(
- add_stdout_handler: bool = True,
- simulation_date_getter: SimulationDateGetter = _mock_simulation_date_getter,
- root_level: LogLevel = WARNING,
- stdout_handler_level: LogLevel = DEBUG,
- formatter: _logging.Formatter = _DEFAULT_FORMATTER,
-) -> None:
- """Initialise logging system and set up root `tlo` logger.
-
- :param add_stdout_handler: Whether to add a handler to output log entries to stdout.
- :param simulation_date_getter: Zero-argument function returning simulation date as
- string in ISO format to use in log entries. Defaults to function returning a
- a fixed dummy date for use before a simulation has been initialised.
- :param root_level: Logging level for root `tlo` logger.
- :param formatter: Formatter to use for logging to stdout.
- """
- global _get_simulation_date, _loggers
- _get_simulation_date = simulation_date_getter
- for logger in _loggers.values():
- logger.reset_attributes()
- root_logger = getLogger("tlo")
- root_logger.setLevel(root_level)
- if add_stdout_handler:
- handler = _logging.StreamHandler(sys.stdout)
- handler.setLevel(stdout_handler_level)
- handler.setFormatter(formatter)
- root_logger.handlers = [
- h
- for h in root_logger.handlers
- if not (isinstance(h, _logging.StreamHandler) and h.stream is sys.stdout)
- ]
- root_logger.addHandler(handler)
-
-
-def reset():
- """Reset global logging state to values at initial import."""
- global _get_simulation_date, _loggers
- while len(_loggers) > 0:
- name, _ = _loggers.popitem()
- _logging.root.manager.loggerDict.pop(name, None) # pylint: disable=E1101
- _loggers.clear()
- _get_simulation_date = _mock_simulation_date_getter
-
-
-def set_output_file(
- log_path: Path,
- formatter: _logging.Formatter = _DEFAULT_FORMATTER,
-) -> _logging.FileHandler:
- """Add file handler to logger.
-
- :param log_path: Path for file.
- :return: File handler object.
- """
- file_handler = _logging.FileHandler(log_path)
- file_handler.setFormatter(formatter)
- logger = getLogger("tlo")
- logger.handlers = [
- h for h in logger.handlers if not isinstance(h, _logging.FileHandler)
- ]
- logger.addHandler(file_handler)
- return file_handler
-
-
-def disable(level: LogLevel) -> None:
- """Disable all logging calls of specified level and below."""
+def disable(level):
_logging.disable(level)
-def getLogger(name: str = "tlo") -> Logger:
+def getLogger(name='tlo'):
"""Returns a TLO logger of the specified name"""
- if name not in _loggers:
- _loggers[name] = Logger(name)
- return _loggers[name]
-
-
-def _numeric_or_str_sort_key(value):
- """Key function to sort mixture of numeric and string items.
-
- Orders non-string values first and then string values, assuming ascending order.
- """
- return isinstance(value, str), value
-
-
-def _convert_keys_to_strings_and_sort(data: dict) -> dict[str, Any]:
- """Convert all dictionary keys to strings and sort dictionary by key."""
- # Sort by mix of numeric or string keys _then_ convert all keys to strings to
- # ensure stringified numeric keys have natural numeric ordering, for example
- # '1', '2', '10' not '1', '10', '2'
- sorted_data = dict(
- (str(k), v)
- for k, v in sorted(data.items(), key=lambda i: _numeric_or_str_sort_key(i[0]))
- )
- if len(sorted_data) != len(data):
- raise ValueError(
- f"At least one pair of keys in data dictionary {data} map to same string."
- )
- return sorted_data
-
-
-def _sort_set_with_numeric_or_str_elements(data: set) -> list:
- """Sort a set with elements that may be either strings or numeric types."""
- return sorted(data, key=_numeric_or_str_sort_key)
-
-
-def _get_log_data_as_dict(data: LogData) -> dict:
- """Convert log data to a dictionary if it isn't already"""
- if isinstance(data, dict):
- return _convert_keys_to_strings_and_sort(data)
- if isinstance(data, pd.DataFrame):
- if len(data) == 1:
- data_dict = data.iloc[0].to_dict()
- return _convert_keys_to_strings_and_sort(data_dict)
- else:
- raise ValueError(
- "Logging multirow dataframes is not currently supported - "
- "if you need this feature let us know"
- )
- if isinstance(data, (list, set, tuple, pd.Series)):
- if isinstance(data, set):
- data = _sort_set_with_numeric_or_str_elements(data)
- return {f"item_{index + 1}": value for index, value in enumerate(data)}
- if isinstance(data, str):
- return {"message": data}
- raise ValueError(f"Unexpected type given as data:\n{data}")
-
-
-def _convert_numpy_scalars_to_python_types(data: dict) -> dict:
- """Convert NumPy scalar types to suitable standard Python types."""
- return {
- key: (
- value.item() if isinstance(value, (np.number, np.bool_, np.str_)) else value
- )
- for key, value in data.items()
- }
-
-
-def _get_columns_from_data_dict(data: dict) -> dict:
- """Get columns dictionary specifying types of data dictionary values."""
- # using type().__name__ so both pandas and stdlib types can be used
- return {k: type(v).__name__ for k, v, in data.items()}
+ if name not in _LOGGERS:
+ _LOGGERS[name] = Logger(name)
+ return _LOGGERS[name]
-class Logger:
- """Logger for structured log messages output by simulation.
+class _MockSim:
+ # used as place holder for any logging that happens before simulation is setup!
+ class MockDate:
+ @staticmethod
+ def isoformat():
+ return "0000-00-00T00:00:00"
+ date = MockDate()
- Outputs structured log messages in JSON format along with simulation date log entry
- was generated at. Log messages are associated with a string key and for each key
- the log message data is expected to have a fixed structure:
- - Collection like data (tuples, lists, sets) should be of fixed length.
- - Mapping like data (dictionaries, pandas series and dataframes) should have a fixed
- set of keys and the values should be of fixed data types.
+class Logger:
+ """A Logger for TLO log messages, with simplified usage. Outputs structured log messages in JSON
+ format and is connected to the Simulation instance."""
+ HASH_LEN = 10
- The first log message for a given key will generate a 'header' log entry which
- records the structure of the message with subsequent log messages only logging the
- values for efficiency, hence the requirement for the structure to remain fixed.
- """
+ def __init__(self, name: str, level=_logging.NOTSET):
- HASH_LEN = 10
+ assert name.startswith('tlo'), f'Only logging of tlo modules is allowed; name is {name}'
- def __init__(self, name: str, level: LogLevel = _DEFAULT_LEVEL) -> None:
- assert name.startswith(
- "tlo"
- ), f"Only logging of tlo modules is allowed; name is {name}"
# we build our logger on top of the standard python logging
self._std_logger = _logging.getLogger(name=name)
self._std_logger.setLevel(level)
- # don't propagate messages up from "tlo" to root logger
- if name == "tlo":
+ self.name = self._std_logger.name
+
+ # don't propograte messages up from "tlo" to root logger
+ if name == 'tlo':
self._std_logger.propagate = False
- # the unique identifiers of the structured logging calls for this logger
- self._uuids = dict()
- # the columns for the structured logging calls for this logger
- self._columns = dict()
- def __repr__(self) -> str:
- return f""
+ # the key of the structured logging calls for this logger
+ self.keys = dict()
- @property
- def name(self) -> str:
- return self._std_logger.name
+ # populated by init_logging(simulation) for the top-level "tlo" logger
+ self.simulation = _MockSim()
+
+ # a logger should only be using old-style or new-style logging, not a mixture
+ self.logged_stdlib = False
+ self.logged_structured = False
+
+ # disable logging multirow dataframes until we're confident it's robust
+ self._disable_dataframe_logging = True
+
+ def __repr__(self):
+ return f''
@property
- def handlers(self) -> List[_logging.Handler]:
+ def handlers(self):
return self._std_logger.handlers
@property
- def level(self) -> LogLevel:
+ def level(self):
return self._std_logger.level
@handlers.setter
- def handlers(self, handlers: List[_logging.Handler]):
+ def handlers(self, handlers):
self._std_logger.handlers.clear()
for handler in handlers:
self._std_logger.handlers.append(handler)
- def addHandler(self, hdlr: _logging.Handler):
+ def addHandler(self, hdlr):
self._std_logger.addHandler(hdlr=hdlr)
- def isEnabledFor(self, level: LogLevel) -> bool:
+ def isEnabledFor(self, level):
return self._std_logger.isEnabledFor(level)
- def reset_attributes(self) -> None:
+ def reset_attributes(self):
"""Reset logger attributes to an unset state"""
# clear all logger settings
self.handlers.clear()
- self._uuids.clear()
- self._columns.clear()
- self.setLevel(_DEFAULT_LEVEL)
-
- def setLevel(self, level: LogLevel) -> None:
+ self.keys.clear()
+ self.simulation = _MockSim()
+ # boolean attributes used for now, can be removed after transition to structured logging
+ self.logged_stdlib = False
+ self.logged_structured = False
+ self.setLevel(INFO)
+
+ def setLevel(self, level):
self._std_logger.setLevel(level)
- def _get_uuid(self, key: str) -> str:
- hexdigest = hashlib.md5(f"{self.name}+{key}".encode()).hexdigest()
- return hexdigest[: Logger.HASH_LEN]
-
- def _get_json(
- self,
- level: int,
- key: str,
- data: Optional[LogData] = None,
- description: Optional[str] = None,
- ) -> str:
- """Writes structured log message if handler allows this and level is allowed.
-
- Will write a header line the first time a new logging key is encountered.
- Then will only write data rows in later rows for this logging key.
-
- :param level: Level the message is being logged as.
- :param key: Logging key.
- :param data: Data to be logged.
- :param description: Description of this log type.
-
- :returns: String with JSON-encoded data row and optionally header row.
+ def _get_data_as_dict(self, data):
+ """Convert log data to a dictionary if it isn't already"""
+ if isinstance(data, dict):
+ return data
+ if isinstance(data, pd.DataFrame):
+ if len(data.index) == 1:
+ return data.to_dict('records')[0]
+ elif self._disable_dataframe_logging:
+ raise ValueError("Logging multirow dataframes is disabled - if you need this feature let us know")
+ else:
+ return {'dataframe': data.to_dict('index')}
+ if isinstance(data, (list, set, tuple, pd.Series)):
+ return {f'item_{index + 1}': value for index, value in enumerate(data)}
+ if isinstance(data, str):
+ return {'message': data}
+
+ raise ValueError(f'Unexpected type given as data:\n{data}')
+
+ def _get_json(self, level, key, data: Union[dict, pd.DataFrame, list, set, tuple, str] = None, description=None):
+ """Writes structured log message if handler allows this and logging level is allowed
+
+ Will write a header line the first time a new logging key is encountered
+ Then will only write data rows in later rows for this logging key
+
+ :param level: Level the message is being logged as
+ :param key: logging key
+ :param data: data to be logged
+ :param description: description of this log type
"""
- data = _get_log_data_as_dict(data)
- data = _convert_numpy_scalars_to_python_types(data)
- header_json = None
+ # message level less than than the logger level, early exit
+ if level < self._std_logger.level:
+ return
+
+ data = self._get_data_as_dict(data)
+ header_json = ""
- if key not in self._uuids:
+ if key not in self.keys:
# new log key, so create header json row
- uuid = self._get_uuid(key)
- columns = _get_columns_from_data_dict(data)
- self._uuids[key] = uuid
- self._columns[key] = columns
+ uuid = hashlib.md5(f"{self.name}+{key}".encode()).hexdigest()[:Logger.HASH_LEN]
+ self.keys[key] = uuid
+
header = {
"uuid": uuid,
"type": "header",
"module": self.name,
"key": key,
"level": _logging.getLevelName(level),
- "columns": columns,
- "description": description,
+ # using type().__name__ so both pandas and stdlib types can be used
+ "columns": {key: type(value).__name__ for key, value in data.items()},
+ "description": description
}
- header_json = json.dumps(header)
- else:
- uuid = self._uuids[key]
- columns = _get_columns_from_data_dict(data)
- if columns != self._columns[key]:
- header_columns = set(self._columns[key].items())
- logged_columns = set(columns.items())
- msg = (
- f"Inconsistent columns in logged values for {self.name} logger "
- f"with key {key} compared to header generated from initial log "
- f"entry:\n"
- f" Columns in header not in logged values are\n"
- f" {dict(sorted(header_columns - logged_columns))}\n"
- f" Columns in logged values not in header are\n"
- f" {dict(sorted(logged_columns - header_columns))}"
- )
- warnings.warn(
- msg,
- InconsistentLoggedColumnsWarning,
- # Set stack level so that user is given location of top-level
- # {info,warning,debug,critical} convenience method call
- stacklevel=3,
- )
-
- # create data json row
- row = {
- "uuid": uuid,
- "date": _get_simulation_date(),
- "values": list(data.values()),
- }
+ header_json = json.dumps(header) + "\n"
+
+ uuid = self.keys[key]
+
+ # create data json row; in DEBUG mode we echo the module and key for easier eyeballing
if self._std_logger.level == DEBUG:
- # in DEBUG mode we echo the module and key for easier eyeballing
- row["module"] = self.name
- row["key"] = key
+ row = {"date": getLogger('tlo').simulation.date.isoformat(),
+ "module": self.name,
+ "key": key,
+ "uuid": uuid,
+ "values": list(data.values())}
+ else:
+ row = {"uuid": uuid,
+ "date": getLogger('tlo').simulation.date.isoformat(),
+ "values": list(data.values())}
row_json = json.dumps(row, cls=encoding.PandasEncoder)
- return row_json if header_json is None else f"{header_json}\n{row_json}"
-
- def log(
- self,
- level: LogLevel,
- key: str,
- data: LogData,
- description: Optional[str] = None,
- ) -> None:
- """Log structured data for a key at specified level with optional description.
-
- :param level: Level the message is being logged as.
- :param key: Logging key.
- :param data: Data to be logged.
- :param description: Description of this log type.
- """
+ return f"{header_json}{row_json}"
+
+ def _make_old_style_msg(self, level, msg):
+ return f'{level}|{self.name}|{msg}'
+
+ def _check_logging_style(self, is_structured: bool):
+ """Set booleans for logging type and throw exception if both types of logging haven't been used"""
+ if is_structured:
+ self.logged_structured = True
+ else:
+ self.logged_stdlib = True
+
+ if self.logged_structured and self.logged_stdlib:
+ raise ValueError(f"Both oldstyle and structured logging has been used for {self.name}, "
+ "please update all logging to use structured logging")
+
+ def _check_and_filter(self, msg=None, *args, key=None, data=None, description=None, level, **kwargs):
if self._std_logger.isEnabledFor(level):
- msg = self._get_json(
- level=level, key=key, data=data, description=description
- )
- self._std_logger.log(level=level, msg=msg)
-
- critical = partialmethod(log, CRITICAL)
- debug = partialmethod(log, DEBUG)
- info = partialmethod(log, INFO)
- warning = partialmethod(log, WARNING)
+ level_str = _logging.getLevelName(level) # e.g. 'CRITICAL', 'INFO' etc.
+ level_function = getattr(self._std_logger, level_str.lower()) # e.g. `critical` or `info` methods
+ if key is None or data is None:
+ raise ValueError("Structured logging requires `key` and `data` keyword arguments")
+ self._check_logging_style(is_structured=True)
+ level_function(self._get_json(level=level, key=key, data=data, description=description))
+
+ def critical(self, msg=None, *args, key: str = None,
+ data: Union[dict, pd.DataFrame, list, set, tuple, str] = None, description=None, **kwargs):
+ self._check_and_filter(msg, *args, key=key, data=data, description=description, level=CRITICAL, **kwargs)
+
+ def debug(self, msg=None, *args, key: str = None,
+ data: Union[dict, pd.DataFrame, list, set, tuple, str] = None, description=None, **kwargs):
+ self._check_and_filter(msg, *args, key=key, data=data, description=description, level=DEBUG, **kwargs)
+
+ def info(self, msg=None, *args, key: str = None,
+ data: Union[dict, pd.DataFrame, list, set, tuple, str] = None, description=None, **kwargs):
+ self._check_and_filter(msg, *args, key=key, data=data, description=description, level=INFO, **kwargs)
+
+ def warning(self, msg=None, *args, key: str = None,
+ data: Union[dict, pd.DataFrame, list, set, tuple, str] = None, description=None, **kwargs):
+ self._check_and_filter(msg, *args, key=key, data=data, description=description, level=WARNING, **kwargs)
+
+
+CRITICAL = _logging.CRITICAL
+DEBUG = _logging.DEBUG
+FATAL = _logging.FATAL
+INFO = _logging.INFO
+WARNING = _logging.WARNING
+
+_FORMATTER = _logging.Formatter('%(message)s')
+_LOGGERS = {'tlo': Logger('tlo', WARNING)}
diff --git a/src/tlo/logging/encoding.py b/src/tlo/logging/encoding.py
index c5db27caa5..9968ce9cb8 100644
--- a/src/tlo/logging/encoding.py
+++ b/src/tlo/logging/encoding.py
@@ -2,7 +2,6 @@
import numpy as np
import pandas as pd
-from pandas.api.types import is_extension_array_dtype
class PandasEncoder(json.JSONEncoder):
@@ -11,16 +10,16 @@ def default(self, obj):
# using base classes for numpy numeric types
if isinstance(obj, np.floating):
return float(obj)
- elif isinstance(obj, np.integer):
+ elif isinstance(obj, np.signedinteger):
return int(obj)
elif isinstance(obj, pd.Timestamp):
return obj.isoformat()
- elif is_extension_array_dtype(obj):
- # for pandas extension dtypes assume length 1 arrays / series are scalars
- return obj.tolist()[0 if len(obj) == 1 else slice(None)]
+ elif isinstance(obj, pd.Categorical):
+ # assume only only one categorical value per cell
+ return obj.tolist()[0]
elif isinstance(obj, set):
return list(obj)
- elif isinstance(obj, (type(pd.NaT), type(pd.NA))):
+ elif isinstance(obj, type(pd.NaT)):
return None
# when logging a series directly, numpy datatypes are used
elif isinstance(obj, np.datetime64):
diff --git a/src/tlo/logging/helpers.py b/src/tlo/logging/helpers.py
index 99fc51c473..2195c602d0 100644
--- a/src/tlo/logging/helpers.py
+++ b/src/tlo/logging/helpers.py
@@ -1,14 +1,26 @@
import logging as _logging
-from collections.abc import Collection, Iterable
-from typing import Dict, List, Optional, Union
+import sys
+from pathlib import Path
+from typing import Dict
-import pandas as pd
-from pandas.api.types import is_extension_array_dtype
+from .core import _FORMATTER, _LOGGERS, DEBUG, getLogger
-from .core import getLogger
+def set_output_file(log_path: Path) -> _logging.FileHandler:
+ """Add filehandler to logger
-def set_logging_levels(custom_levels: Dict[str, int]) -> None:
+ :param log_path: path for file
+ :return: filehandler object
+ """
+ file_handler = _logging.FileHandler(log_path)
+ file_handler.setFormatter(_FORMATTER)
+ getLogger('tlo').handlers = [h for h in getLogger('tlo').handlers
+ if not isinstance(h, _logging.FileHandler)]
+ getLogger('tlo').addHandler(file_handler)
+ return file_handler
+
+
+def set_logging_levels(custom_levels: Dict[str, int]):
"""Set custom logging levels for disease modules
:param custom_levels: Dictionary of modules and their level, '*' can be used as a key for all modules
@@ -53,78 +65,23 @@ def set_logging_levels(custom_levels: Dict[str, int]) -> None:
getLogger(logger_name).setLevel(logger_level)
-def get_dataframe_row_as_dict_for_logging(
- dataframe: pd.DataFrame,
- row_label: Union[int, str],
- columns: Optional[Iterable[str]] = None,
-) -> dict:
- """Get row of a pandas dataframe in a format suitable for logging.
-
- Retrieves entries for all or a subset of columns for a particular row in a dataframe
- and returns a dict keyed by column name, with values NumPy or pandas extension types
- which should be the same for all rows in dataframe.
-
- :param dataframe: Population properties dataframe to get properties from.
- :param row_label: Unique index label identifying row in dataframe.
- :param columns: Set of column names to extract - if ``None``, the default, all
- column values will be returned.
- :returns: Dictionary with column names as keys and corresponding entries in row as
- values.
- """
- dataframe = dataframe.convert_dtypes(convert_integer=False, convert_floating=False)
- columns = dataframe.columns if columns is None else columns
- row_index = dataframe.index.get_loc(row_label)
- return {
- column_name:
- dataframe[column_name].values[row_index]
- # pandas extension array datatypes such as nullable types and categoricals, will
- # be type unstable if a scalar is returned as NA / NaT / NaN entries will have a
- # different type from non-missing entries, therefore use a length 1 array of
- # relevant NumPy or pandas extension type in these cases to ensure type
- # stability across different rows.
- if not is_extension_array_dtype(dataframe[column_name].dtype) else
- dataframe[column_name].values[row_index:row_index+1]
- for column_name in columns
- }
+def init_logging(add_stdout_handler=True):
+ """Initialise default logging with stdout stream"""
+ for logger_name, logger in _LOGGERS.items():
+ logger.reset_attributes()
+ if add_stdout_handler:
+ handler = _logging.StreamHandler(sys.stdout)
+ handler.setLevel(DEBUG)
+ handler.setFormatter(_FORMATTER)
+ getLogger('tlo').addHandler(handler)
+ _logging.basicConfig(level=_logging.WARNING)
-def grouped_counts_with_all_combinations(
- dataframe: pd.DataFrame,
- group_by_columns: List[str],
- column_possible_values: Optional[Dict[str, Collection]] = None,
-) -> pd.Series:
- """Perform group-by count in which all combinations of column values are included.
-
- As all combinations are included irrespective of whether they are present in data
- (and so have a non-zero count), this gives a multi-index series output of fixed
- structure suitable for logging.
-
- Attempts to convert all columns to categorical datatype, with bool(ean) columns
- automatically converted, and other non-categorical columns needing to have set of
- possible values specified (which requires that this set is finite).
-
- :param dataframe: Dataframe to perform group-by counts on.
- :param group_by_columns: Columns to perform grouping on.
- :param column_possible_values: Dictionary mapping from column names to set of
- possible values for all columns not of categorical or bool(ean) data type.
- :returns: Multi-index series with values corresponding to grouped counts.
+def set_simulation(simulation):
+ """
+ Inject simulation into logger for structured logging, called by the simulation
+ :param simulation:
+ :return:
"""
- subset = dataframe[group_by_columns].copy()
- # Convert any bool(ean) columns to categoricals
- for column_name in group_by_columns:
- if subset[column_name].dtype in ("bool", "boolean"):
- subset[column_name] = pd.Categorical(
- subset[column_name], categories=[True, False]
- )
- # For other non-categorical columns possible values need to be explicitly stated
- if column_possible_values is not None:
- for column_name, possible_values in column_possible_values.items():
- subset[column_name] = pd.Categorical(
- subset[column_name], categories=possible_values
- )
- if not (subset.dtypes == "category").all():
- msg = "At least one column not convertable to categorical dtype:\n" + str(
- {subset.dtypes[subset.dtypes != "categorical"]}
- )
- raise ValueError(msg)
- return subset.groupby(by=group_by_columns).size()
+ logger = getLogger('tlo')
+ logger.simulation = simulation
diff --git a/src/tlo/methods/alri.py b/src/tlo/methods/alri.py
index 70ac14fe2d..277726e0ff 100644
--- a/src/tlo/methods/alri.py
+++ b/src/tlo/methods/alri.py
@@ -21,6 +21,7 @@
"""
from __future__ import annotations
+import types
from collections import defaultdict
from itertools import chain
from pathlib import Path
@@ -30,18 +31,17 @@
import pandas as pd
from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata
from tlo.methods.causes import Cause
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
from tlo.util import random_date, sample_outcome
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
@@ -55,7 +55,7 @@
# ---------------------------------------------------------------------------------------------------------
-class Alri(Module, GenericFirstAppointmentsMixin):
+class Alri(Module):
"""This is the disease module for Acute Lower Respiratory Infections."""
INIT_DEPENDENCIES = {
@@ -1014,85 +1014,139 @@ def look_up_consumables(self):
get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name
- # # # # # # Dosages by weight # # # # # #
- # Assuming average weight of 0-5 is 12kg (abstraction). Doses sourced for WHO Pocket book of hospital care for
- # children: Second edition 2014
+ def get_dosage_for_age_in_months(age_in_whole_months: float, doses_by_age_in_months: Dict[int, float]):
+ """Returns the dose corresponding to age, using the lookup provided in `doses`. The format of `doses` is:
+ {: }."""
+
+ for upper_age_bound_in_months, _dose in sorted(doses_by_age_in_months.items()):
+ if age_in_whole_months < upper_age_bound_in_months:
+ return _dose
+ return _dose
+
+ # # # # # # Dosages by age # # # # # #
# Antibiotic therapy -------------------
- # Antibiotics for non-severe pneumonia - oral amoxicillin for 5 days (40mg/kg BD - ((12*40)*2)*5 =4800mg)
+ # Antibiotics for non-severe pneumonia - oral amoxicillin for 5 days
self.consumables_used_in_hsi['Amoxicillin_tablet_or_suspension_5days'] = {
- get_item_code(item='Amoxycillin 250mg_1000_CMST'): 4800,
- get_item_code(item='Amoxycillin 125mg/5ml suspension, PFR_0.025_CMST'): 192} # 25mg/ml - 4800/25
+ get_item_code(item='Amoxycillin 250mg_1000_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {2: 0, 12: 0.006, 36: 0.012, np.inf: 0.018}
+ ),
+ get_item_code(item='Amoxycillin 125mg/5ml suspension, PFR_0.025_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {2: 0, 12: 1, 36: 2, np.inf: 3}
+ ),
+ }
- # Antibiotics for non-severe pneumonia - oral amoxicillin for 3 days (40mg/kg BD - ((12*40)*2)*3 =2880mg)
+ # Antibiotics for non-severe pneumonia - oral amoxicillin for 3 days
self.consumables_used_in_hsi['Amoxicillin_tablet_or_suspension_3days'] = {
- get_item_code(item='Amoxycillin 250mg_1000_CMST'): 2880,
- get_item_code(item='Amoxycillin 125mg/5ml suspension, PFR_0.025_CMST'): 115} # 25mg/ml - 2880/25
+ get_item_code(item='Amoxycillin 250mg_1000_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {2: 0, 12: 0.01, 36: 0.02, np.inf: 0.03}
+ ),
+ get_item_code(item='Amoxycillin 125mg/5ml suspension, PFR_0.025_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {2: 0, 12: 1, 36: 2, np.inf: 3}
+ ),
+ }
- # Antibiotics for non-severe pneumonia - oral amoxicillin for 7 days for young infants only (40mg/kg BD -
- # ((12*40)*2)*7 =6720mg)
+ # Antibiotics for non-severe pneumonia - oral amoxicillin for 7 days for young infants only
self.consumables_used_in_hsi['Amoxicillin_tablet_or_suspension_7days'] = {
- get_item_code(item='Amoxycillin 250mg_1000_CMST'): 6720,
- get_item_code(item='Amoxycillin 125mg/5ml suspension, PFR_0.025_CMST'): 269} # 25mg/ml - 6720/25
+ get_item_code(item='Amoxycillin 250mg_1000_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {1: 0.004, 2: 0.006, np.inf: 0.01}
+ ),
+ get_item_code(item='Amoxycillin 125mg/5ml suspension, PFR_0.025_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {1: 0.4, 2: 0.5, np.inf: 1}
+ ),
+ }
# Antibiotic therapy for severe pneumonia - ampicillin package
- # Amp. dose - 50mg/KG QDS 5 days = (50*12)*4)*5 = 12_000mg
- # Gent. dose -7.5mg/kg per day 5 days = (7.5*12)*5 = 450mg
self.consumables_used_in_hsi['1st_line_IV_antibiotics'] = {
- get_item_code(item='Ampicillin injection 500mg, PFR_each_CMST'): 24, # 500mg vial -12_000/500
- get_item_code(item='Gentamicin Sulphate 40mg/ml, 2ml_each_CMST'): 6, # 80mg/2ml = 450/8
+ get_item_code(item='Ampicillin injection 500mg, PFR_each_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {1: 3.73, 2: 5.6, 4: 8, 12: 16, 36: 24, np.inf: 40}
+ ),
+ get_item_code(item='Gentamicin Sulphate 40mg/ml, 2ml_each_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {1: 0.7, 2: 1.4, 4: 2.81, 12: 4.69, 36: 7.03, np.inf: 9.37}
+ ),
get_item_code(item='Cannula iv (winged with injection pot) 16_each_CMST'): 1,
get_item_code(item='Syringe, Autodisable SoloShot IX '): 1
}
# # Antibiotic therapy for severe pneumonia - benzylpenicillin package when ampicillin is not available
- # Benpen dose - 50_000IU/KG QDS 5 days = (50_000*12)*4)*5 = 12_000_000IU = 8g (approx)
- # Gent. dose -7.5mg/kg per day 5 days = (7.5*12)*5 = 450mg
self.consumables_used_in_hsi['Benzylpenicillin_gentamicin_therapy_for_severe_pneumonia'] = {
- get_item_code(item='Benzylpenicillin 3g (5MU), PFR_each_CMST'): 8,
- get_item_code(item='Gentamicin Sulphate 40mg/ml, 2ml_each_CMST'): 6, # 80mg/2ml = 450/8
+ get_item_code(item='Benzylpenicillin 3g (5MU), PFR_each_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {1: 2, 2: 5, 4: 8, 12: 15, 36: 24, np.inf: 34}
+ ),
+ get_item_code(item='Gentamicin Sulphate 40mg/ml, 2ml_each_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {1: 0.7, 2: 1.4, 4: 2.81, 12: 4.69, 36: 7.03, np.inf: 9.37}
+ ),
get_item_code(item='Cannula iv (winged with injection pot) 16_each_CMST'): 1,
get_item_code(item='Syringe, Autodisable SoloShot IX '): 1
}
# Second line of antibiotics for severe pneumonia, if Staph not suspected
- # Ceft. dose = 80mg/kg per day 5 days = (80*12)*5 = 4800mg
self.consumables_used_in_hsi['Ceftriaxone_therapy_for_severe_pneumonia'] = {
- get_item_code(item='Ceftriaxone 1g, PFR_each_CMST'): 1, # smallest unit is 1g
+ get_item_code(item='Ceftriaxone 1g, PFR_each_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {4: 1.5, 12: 3, 36: 5, np.inf: 7}
+ ),
get_item_code(item='Cannula iv (winged with injection pot) 16_each_CMST'): 1,
get_item_code(item='Syringe, Autodisable SoloShot IX '): 1
}
# Second line of antibiotics for severe pneumonia, if Staph is suspected
- # Flucox. dose = 50mg/kg QDS 7 days = ((50*12)*4)*7 = 16_800mg
- # Oral flucox dose. = same
self.consumables_used_in_hsi['2nd_line_Antibiotic_therapy_for_severe_staph_pneumonia'] = {
- get_item_code(item='Flucloxacillin 250mg, vial, PFR_each_CMST'): 16_800,
- get_item_code(item='Gentamicin Sulphate 40mg/ml, 2ml_each_CMST'): 6, # 80mg/2ml = 450/8
+ get_item_code(item='Flucloxacillin 250mg, vial, PFR_each_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {2: 21, 4: 22.4, 12: 37.3, 36: 67.2, 60: 93.3, np.inf: 140}
+ ),
+ get_item_code(item='Gentamicin Sulphate 40mg/ml, 2ml_each_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {4: 2.81, 12: 4.69, 36: 7.03, 60: 9.37, np.inf: 13.6}
+ ),
get_item_code(item='Cannula iv (winged with injection pot) 16_each_CMST'): 1,
get_item_code(item='Syringe, Autodisable SoloShot IX '): 1,
- get_item_code(item='Flucloxacillin 250mg_100_CMST'): 16_800}
+ get_item_code(item='Flucloxacillin 250mg_100_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {4: 0.42, 36: 0.84, 60: 1.68, np.inf: 1.68}
+ ),
+ }
# First dose of antibiotic before referral -------------------
- # single dose of 7.5mg gent and 50mg/g amp. given
+
# Referral process in iCCM for severe pneumonia, and at health centres for HIV exposed/infected
self.consumables_used_in_hsi['First_dose_oral_amoxicillin_for_referral'] = {
- get_item_code(item='Amoxycillin 250mg_1000_CMST'): 250,
+ get_item_code(item='Amoxycillin 250mg_1000_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {12: 0.001, 36: 0.002, np.inf: 0.003}
+ ),
}
# Referral process at health centres for severe cases
self.consumables_used_in_hsi['First_dose_IM_antibiotics_for_referral'] = {
- get_item_code(item='Ampicillin injection 500mg, PFR_each_CMST'): 2, # 2 x 500mg vial
- get_item_code(item='Gentamicin Sulphate 40mg/ml, 2ml_each_CMST'): 2, # assuming single dose at referral
+ get_item_code(item='Ampicillin injection 500mg, PFR_each_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {4: 0.4, 12: 0.8, 36: 1.4, np.inf: 2}
+ ),
+ get_item_code(item='Gentamicin Sulphate 40mg/ml, 2ml_each_CMST'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {4: 0.56, 12: 0.94, 36: 1.41, np.inf: 1.87}
+ ),
get_item_code(item='Cannula iv (winged with injection pot) 16_each_CMST'): 1,
get_item_code(item='Syringe, Autodisable SoloShot IX '): 1
}
# Oxygen, pulse oximetry and x-ray -------------------
- # Oxygen for hypoxaemia - 5/l per min (Approx) for 3 days ((24*60)*5)*3
+ # Oxygen for hypoxaemia
self.consumables_used_in_hsi['Oxygen_Therapy'] = {
- get_item_code(item='Oxygen, 1000 liters, primarily with oxygen cylinders'): 21_600,
+ get_item_code(item='Oxygen, 1000 liters, primarily with oxygen cylinders'): 1,
}
# Pulse oximetry
@@ -1108,7 +1162,10 @@ def look_up_consumables(self):
# Optional consumables -------------------
# Paracetamol
self.consumables_used_in_hsi['Paracetamol_tablet'] = {
- get_item_code(item='Paracetamol, tablet, 100 mg'): 240, # 20mg/kg
+ get_item_code(item='Paracetamol, tablet, 100 mg'):
+ lambda _age: get_dosage_for_age_in_months(int(_age * 12.0),
+ {36: 12, np.inf: 18}
+ ),
}
# Maintenance of fluids via nasograstric tube
@@ -1121,6 +1178,11 @@ def look_up_consumables(self):
get_item_code(item='Salbutamol sulphate 1mg/ml, 5ml_each_CMST'): 2
}
+ # Bronchodilator - oral
+ self.consumables_used_in_hsi['Oral_Brochodilator'] = {
+ get_item_code(item='Salbutamol, syrup, 2 mg/5 ml'): 1,
+ get_item_code(item='Salbutamol, tablet, 4 mg'): 1
+ }
def end_episode(self, person_id):
"""End the episode infection for a person (i.e. reset all properties to show no current infection or
@@ -1253,7 +1315,7 @@ def do_effects_of_treatment_and_return_outcome(self, person_id, antibiotic_provi
# Gather underlying properties that will affect success of treatment
SpO2_level = person.ri_SpO2_level
- symptoms = self.sim.modules['SymptomManager'].has_what(person_id=person_id)
+ symptoms = self.sim.modules['SymptomManager'].has_what(person_id)
imci_symptom_based_classification = self.get_imci_classification_based_on_symptoms(
child_is_younger_than_2_months=person.age_exact_years < (2.0 / 12.0),
symptoms=symptoms,
@@ -1363,35 +1425,39 @@ def _ultimate_treatment_indicated_for_patient(classification_for_treatment_decis
def do_at_generic_first_appt(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
facility_level: str,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
# Action taken when a child (under 5 years old) presents at a
# generic appointment (emergency or non-emergency) with symptoms
# of `cough` or `difficult_breathing`.
- if individual_properties["age_years"] <= 5 and (
+ if patient_details.age_years <= 5 and (
("cough" in symptoms) or ("difficult_breathing" in symptoms)
):
self.record_sought_care_for_alri()
# All persons have an initial out-patient appointment at the current facility level.
event = HSI_Alri_Treatment(
- person_id=person_id, module=self, facility_level=facility_level
+ person_id=patient_id, module=self, facility_level=facility_level
)
- schedule_hsi_event(
+ self.healthsystem.schedule_hsi_event(
event,
topen=self.sim.date,
tclose=self.sim.date + pd.DateOffset(days=1),
priority=1,
)
- def do_at_generic_first_appt_emergency(self, **kwargs) -> None:
+ def do_at_generic_first_appt_emergency(
+ self,
+ **kwargs,
+ ) -> IndividualPropertyUpdates:
# Emergency and non-emergency treatment is identical for alri
- self.do_at_generic_first_appt(**kwargs)
+ return self.do_at_generic_first_appt(
+ **kwargs,
+ )
class Models:
@@ -2354,7 +2420,11 @@ def _get_cons(self, _arg: Union[str, Tuple[str]]) -> bool:
def _get_cons_group(self, item_group_str: str) -> bool:
"""True if _all_ of a group of consumables (identified by a string) is available."""
if item_group_str is not None:
- return self.get_consumables(self.module.consumables_used_in_hsi[item_group_str])
+ return self.get_consumables(
+ item_codes={
+ k: v(self._age_exact_years) if isinstance(v, types.LambdaType) else v
+ for k, v in self.module.consumables_used_in_hsi[item_group_str].items()
+ })
else:
raise ValueError('String for the group of consumables not provided')
@@ -2551,8 +2621,6 @@ def _get_disease_classification_for_treatment_decision(self,
'chest_indrawing_pneumonia', (symptoms-based assessment)
'cough_or_cold' (symptoms-based assessment)
}."""
- if use_oximeter:
- self.add_equipment({'Pulse oximeter'})
child_is_younger_than_2_months = age_exact_years < (2.0 / 12.0)
@@ -2608,15 +2676,6 @@ def _try_treatment(antibiotic_indicated: Tuple[str], oxygen_indicated: bool) ->
oxygen_available = self._get_cons('Oxygen_Therapy')
oxygen_provided = (oxygen_available and oxygen_indicated)
- # If individual is provided with oxygen, add used equipment
- if oxygen_provided:
- self.add_equipment({'Oxygen cylinder, with regulator', 'Nasal Prongs'})
-
- # If individual is provided with intravenous antibiotics, add used equipment
- if antibiotic_provided in ('1st_line_IV_antibiotics',
- 'Benzylpenicillin_gentamicin_therapy_for_severe_pneumonia'):
- self.add_equipment({'Infusion pump', 'Drip stand'})
-
all_things_needed_available = antibiotic_available and (
(oxygen_available and oxygen_indicated) or (not oxygen_indicated)
)
@@ -2698,7 +2757,6 @@ def _provide_bronchodilator_if_wheeze(self, facility_level, symptoms):
if facility_level == '1a':
_ = self._get_cons('Inhaled_Brochodilator')
else:
- # n.b. this is never called, see issue 1172
_ = self._get_cons('Brochodilator_and_Steroids')
def do_on_follow_up_following_treatment_failure(self):
@@ -2706,12 +2764,9 @@ def do_on_follow_up_following_treatment_failure(self):
A further drug will be used but this will have no effect on the chance of the person dying."""
if self._has_staph_aureus():
- cons_avail = self._get_cons('2nd_line_Antibiotic_therapy_for_severe_staph_pneumonia')
+ _ = self._get_cons('2nd_line_Antibiotic_therapy_for_severe_staph_pneumonia')
else:
- cons_avail = self._get_cons('Ceftriaxone_therapy_for_severe_pneumonia')
-
- if cons_avail:
- self.add_equipment({'Infusion pump', 'Drip stand'})
+ _ = self._get_cons('Ceftriaxone_therapy_for_severe_pneumonia')
def apply(self, person_id, squeeze_factor):
"""Assess and attempt to treat the person."""
@@ -2726,7 +2781,7 @@ def apply(self, person_id, squeeze_factor):
return
# Do nothing if the persons does not have indicating symptoms
- symptoms = self.sim.modules['SymptomManager'].has_what(person_id=person_id)
+ symptoms = self.sim.modules['SymptomManager'].has_what(person_id)
if not {'cough', 'difficult_breathing'}.intersection(symptoms):
return self.make_appt_footprint({})
@@ -3009,7 +3064,7 @@ def apply(self, person_id):
assert 'danger_signs_pneumonia' == self.module.get_imci_classification_based_on_symptoms(
child_is_younger_than_2_months=df.at[person_id, 'age_exact_years'] < (2.0 / 12.0),
- symptoms=self.sim.modules['SymptomManager'].has_what(person_id=person_id)
+ symptoms=self.sim.modules['SymptomManager'].has_what(person_id)
)
@@ -3040,7 +3095,7 @@ def apply(self, person_id):
assert 'fast_breathing_pneumonia' == \
self.module.get_imci_classification_based_on_symptoms(
- child_is_younger_than_2_months=False, symptoms=self.sim.modules['SymptomManager'].has_what(person_id=person_id)
+ child_is_younger_than_2_months=False, symptoms=self.sim.modules['SymptomManager'].has_what(person_id)
)
diff --git a/src/tlo/methods/bed_days.py b/src/tlo/methods/bed_days.py
index a47b75b16a..ef501f3b2e 100644
--- a/src/tlo/methods/bed_days.py
+++ b/src/tlo/methods/bed_days.py
@@ -5,12 +5,12 @@
"""
from collections import defaultdict
-from typing import Dict, Literal, Tuple
+from typing import Dict, Tuple
import numpy as np
import pandas as pd
-from tlo import Date, Property, Types, logging
+from tlo import Property, Types, logging
# ---------------------------------------------------------------------------------------------------------
# CLASS DEFINITIONS
@@ -145,40 +145,6 @@ def initialise_beddays_tracker(self, model_to_data_popsize_ratio=1.0):
assert not df.isna().any().any()
self.bed_tracker[bed_type] = df
- def switch_beddays_availability(
- self,
- new_availability: Literal["all", "none", "default"],
- effective_on_and_from: Date,
- model_to_data_popsize_ratio: float = 1.0,
- ) -> None:
- """
- Action to be taken if the beddays availability changes in the middle
- of the simulation.
-
- If bed capacities are reduced below the currently scheduled occupancy,
- inpatients are not evicted from beds and are allowed to remain in the
- bed until they are scheduled to leave. Obviously, no new patients will
- be admitted if there is no room in the new capacities.
-
- :param new_availability: The new bed availability. See __init__ for details.
- :param effective_on_and_from: First day from which the new capacities will be imposed.
- :param model_to_data_popsize_ratio: As in initialise_population.
- """
- # Store new bed availability
- self.availability = new_availability
- # Before we update the bed capacity, we need to store its old values
- # This is because we will need to update the trackers to reflect the new#
- # maximum capacities for each bed type.
- old_max_capacities: pd.DataFrame = self._scaled_capacity.copy()
- # Set the new capacity for beds
- self.set_scaled_capacity(model_to_data_popsize_ratio)
- # Compute the difference between the new max capacities and the old max capacities
- difference_in_max = self._scaled_capacity - old_max_capacities
- # For each tracker, after the effective date, impose the difference on the max
- # number of beds
- for bed_type, tracker in self.bed_tracker.items():
- tracker.loc[effective_on_and_from:] += difference_in_max[bed_type]
-
def on_start_of_day(self):
"""Things to do at the start of each new day:
* Refresh inpatient status
@@ -318,60 +284,6 @@ def issue_bed_days_according_to_availability(self, facility_id: int, footprint:
return available_footprint
- def combine_footprints_for_same_patient(
- self, fp1: Dict[str, int], fp2: Dict[str, int]
- ) -> Dict[str, int]:
- """
- Given two footprints that are due to start on the same day, combine the two footprints by
- overlaying the higher-priority bed over the lower-priority beds.
-
- As an example, given the footprints,
- fp1 = {"bedtype1": 2, "bedtype2": 0}
- fp2 = {"bedtype1": 1, "bedtype2": 6}
-
- where bedtype1 is higher priority than bedtype2, we expect the combined allocation to be
- {"bedtype1": 2, "bedtype2": 5}.
-
- This is because footprints are assumed to run in the order of the bedtypes priority; so
- fp2's second day of being allocated to bedtype2 is overwritten by the higher-priority
- allocation to bedtype1 from fp1. The remaining 5 days are allocated to bedtype2 since
- fp1 does not require a bed after the first 2 days, but fp2 does.
-
- :param fp1: Footprint, to be combined with the other argument.
- :param pf2: Footprint, to be combined with the other argument.
- """
- fp1_length = sum(days for days in fp1.values())
- fp2_length = sum(days for days in fp2.values())
- max_length = max(fp1_length, fp2_length)
-
- # np arrays where each entry is the priority of bed allocated by the footprint
- # on that day. fp_priority[i] = priority of the bed allocated by the footprint on
- # day i (where the current day is day 0).
- # By default, fill with priority equal to the lowest bed priority; though all
- # the values will have been explicitly overwritten after the next loop completes.
- fp1_priority = np.ones((max_length,), dtype=int) * (len(self.bed_types) - 1)
- fp2_priority = fp1_priority.copy()
-
- fp1_at = 0
- fp2_at = 0
- for priority, bed_type in enumerate(self.bed_types):
- # Bed type priority is dictated by list order, so it is safe to loop here.
- # We will start with the highest-priority bed type and work to the lowest
- fp1_priority[fp1_at:fp1_at + fp1[bed_type]] = priority
- fp1_at += fp1[bed_type]
- fp2_priority[fp2_at:fp2_at + fp2[bed_type]] = priority
- fp2_at += fp2[bed_type]
-
- # Element-wise minimum of the two priority arrays is then the bed to assign
- final_priorities = np.minimum(fp1_priority, fp2_priority)
- # Final footprint is then formed by converting the priorities into blocks of days
- return {
- # Cast to int here since pd.datetime.timedelta doesn't know what to do with
- # np.int64 types
- bed_type: int(sum(final_priorities == priority))
- for priority, bed_type in enumerate(self.bed_types)
- }
-
def impose_beddays_footprint(self, person_id, footprint):
"""This is called to reflect that a new occupancy of bed-days should be recorded:
* Cause to be reflected in the bed_tracker that an hsi_event is being run that will cause bed to be
@@ -399,7 +311,9 @@ def impose_beddays_footprint(self, person_id, footprint):
remaining_footprint = self.get_remaining_footprint(person_id)
# combine the remaining footprint with the new footprint, with days in each bed-type running concurrently:
- combo_footprint = self.combine_footprints_for_same_patient(footprint, remaining_footprint)
+ combo_footprint = {bed_type: max(footprint[bed_type], remaining_footprint[bed_type])
+ for bed_type in self.bed_types
+ }
# remove the old footprint and apply the combined footprint
self.remove_beddays_footprint(person_id)
diff --git a/src/tlo/methods/bladder_cancer.py b/src/tlo/methods/bladder_cancer.py
index 52271f6f16..4c94fd8f51 100644
--- a/src/tlo/methods/bladder_cancer.py
+++ b/src/tlo/methods/bladder_cancer.py
@@ -13,6 +13,7 @@
import pandas as pd
from tlo import DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata
@@ -21,18 +22,16 @@
from tlo.methods.demography import InstantaneousDeath
from tlo.methods.dxmanager import DxTest
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class BladderCancer(Module, GenericFirstAppointmentsMixin):
+class BladderCancer(Module):
"""Bladder Cancer Disease Module"""
def __init__(self, name=None, resourcefilepath=None):
@@ -596,28 +595,27 @@ def report_daly_values(self):
def do_at_generic_first_appt(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
# Only investigate if the patient is not a child
- if individual_properties["age_years"] > 5:
+ if patient_details.age_years > 5:
# Begin investigation if symptoms are present.
if "blood_urine" in symptoms:
event = HSI_BladderCancer_Investigation_Following_Blood_Urine(
- person_id=person_id, module=self
+ person_id=patient_id, module=self
)
- schedule_hsi_event(
+ self.healthsystem.schedule_hsi_event(
event, topen=self.sim.date, priority=0
)
if "pelvic_pain" in symptoms:
event = HSI_BladderCancer_Investigation_Following_pelvic_pain(
- person_id=person_id, module=self
+ person_id=patient_id, module=self
)
- schedule_hsi_event(
+ self.healthsystem.schedule_hsi_event(
event, topen=self.sim.date, priority=0
)
@@ -718,21 +716,21 @@ def apply(self, person_id, squeeze_factor):
return hs.get_blank_appt_footprint()
# Check that this event has been called for someone with the symptom blood_urine
- assert 'blood_urine' in self.sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert 'blood_urine' in self.sim.modules['SymptomManager'].has_what(person_id)
# If the person is already diagnosed, then take no action:
if not pd.isnull(df.at[person_id, "bc_date_diagnosis"]):
return hs.get_blank_appt_footprint()
# Check consumables are available
- cons_avail = self.get_consumables(item_codes=self.module.item_codes_bladder_can['screening_cystoscopy_core'],
- optional_item_codes=self.module.item_codes_bladder_can[
- 'screening_biopsy_endoscopy_cystoscopy_optional'])
+ # TODO: replace with cystoscope
+ cons_avail = self.get_consumables(item_codes=self.module.item_codes_bladder_can['screening_biopsy_core'],
+ optional_item_codes=
+ self.module.item_codes_bladder_can['screening_biopsy_optional'])
if cons_avail:
# Use a biopsy to diagnose whether the person has bladder Cancer
- # If consumables are available update the use of equipment and run the dx_test representing the biopsy
- self.add_equipment({'Cystoscope', 'Ordinary Microscope', 'Ultrasound scanning machine'})
+ # If consumables are available, run the dx_test representing the biopsy
# Use a cystoscope to diagnose whether the person has bladder Cancer:
dx_result = hs.dx_manager.run_dx_test(
@@ -791,21 +789,21 @@ def apply(self, person_id, squeeze_factor):
return hs.get_blank_appt_footprint()
# Check that this event has been called for someone with the symptom pelvic_pain
- assert 'pelvic_pain' in self.sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert 'pelvic_pain' in self.sim.modules['SymptomManager'].has_what(person_id)
# If the person is already diagnosed, then take no action:
if not pd.isnull(df.at[person_id, "bc_date_diagnosis"]):
return hs.get_blank_appt_footprint()
# Check consumables are available
- cons_avail = self.get_consumables(item_codes=self.module.item_codes_bladder_can['screening_cystoscopy_core'],
+ # TODO: replace with cystoscope
+ cons_avail = self.get_consumables(item_codes=self.module.item_codes_bladder_can['screening_biopsy_core'],
optional_item_codes=self.module.item_codes_bladder_can[
- 'screening_biopsy_endoscopy_cystoscopy_optional'])
+ 'screening_biopsy_optional'])
if cons_avail:
# Use a biopsy to diagnose whether the person has bladder Cancer
- # If consumables are available log the use of equipment and run the dx_test representing the biopsy
- self.add_equipment({'Cystoscope', 'Ordinary Microscope', 'Ultrasound scanning machine'})
+ # If consumables are available, run the dx_test representing the biopsy
# Use a cystoscope to diagnose whether the person has bladder Cancer:
dx_result = hs.dx_manager.run_dx_test(
@@ -894,8 +892,7 @@ def apply(self, person_id, squeeze_factor):
self.module.item_codes_bladder_can['treatment_surgery_optional'])
if cons_avail:
- # If consumables are available and the treatment will go ahead - update the equipment
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery'))
+ # If consumables are available and the treatment will go ahead
# Record date and stage of starting treatment
df.at[person_id, "bc_date_treatment"] = self.sim.date
@@ -999,8 +996,7 @@ def apply(self, person_id, squeeze_factor):
item_codes=self.module.item_codes_bladder_can['palliation'])
if cons_available:
- # If consumables are available and the treatment will go ahead - update the equipment
- self.add_equipment({'Infusion pump', 'Drip stand'})
+ # If consumables are available and the treatment will go ahead
# Record the start of palliative care if this is first appointment
if pd.isnull(df.at[person_id, "bc_date_palliative_care"]):
diff --git a/src/tlo/methods/breast_cancer.py b/src/tlo/methods/breast_cancer.py
index a55c6f4930..21347c1f98 100644
--- a/src/tlo/methods/breast_cancer.py
+++ b/src/tlo/methods/breast_cancer.py
@@ -12,6 +12,7 @@
import pandas as pd
from tlo import DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata
@@ -20,18 +21,16 @@
from tlo.methods.demography import InstantaneousDeath
from tlo.methods.dxmanager import DxTest
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class BreastCancer(Module, GenericFirstAppointmentsMixin):
+class BreastCancer(Module):
"""Breast Cancer Disease Module"""
def __init__(self, name=None, resourcefilepath=None):
@@ -573,20 +572,19 @@ def report_daly_values(self):
def do_at_generic_first_appt(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
# If the patient is not a child and symptoms include breast
# lump discernible
- if individual_properties["age_years"] > 5 and "breast_lump_discernible" in symptoms:
+ if patient_details.age_years > 5 and "breast_lump_discernible" in symptoms:
event = HSI_BreastCancer_Investigation_Following_breast_lump_discernible(
- person_id=person_id,
+ person_id=patient_id,
module=self,
)
- schedule_hsi_event(event, topen=self.sim.date, priority=0)
+ self.healthsystem.schedule_hsi_event(event, topen=self.sim.date, priority=0)
# ---------------------------------------------------------------------------------------------------------
@@ -685,7 +683,7 @@ def apply(self, person_id, squeeze_factor):
return hs.get_blank_appt_footprint()
# Check that this event has been called for someone with the symptom breast_lump_discernible
- assert 'breast_lump_discernible' in self.sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert 'breast_lump_discernible' in self.sim.modules['SymptomManager'].has_what(person_id)
# If the person is already diagnosed, then take no action:
if not pd.isnull(df.at[person_id, "brc_date_diagnosis"]):
@@ -696,13 +694,11 @@ def apply(self, person_id, squeeze_factor):
# Check consumables to undertake biopsy are available
cons_avail = self.get_consumables(item_codes=self.module.item_codes_breast_can['screening_biopsy_core'],
optional_item_codes=
- self.module.item_codes_breast_can[
- 'screening_biopsy_endoscopy_cystoscopy_optional'])
+ self.module.item_codes_breast_can['screening_biopsy_optional'])
if cons_avail:
# Use a biopsy to diagnose whether the person has breast Cancer
- # If consumables are available, add the used equipment and run the dx_test representing the biopsy
- self.add_equipment({'Ultrasound scanning machine', 'Ordinary Microscope'})
+ # If consumables are available, run the dx_test representing the biopsy
dx_result = hs.dx_manager.run_dx_test(
dx_tests_to_run='biopsy_for_breast_cancer_given_breast_lump_discernible',
@@ -766,6 +762,8 @@ def apply(self, person_id, squeeze_factor):
df = self.sim.population.props
hs = self.sim.modules["HealthSystem"]
+ # todo: request consumables needed for this
+
if not df.at[person_id, 'is_alive']:
return hs.get_blank_appt_footprint()
@@ -798,9 +796,7 @@ def apply(self, person_id, squeeze_factor):
)
if cons_available:
- # If consumables are available and the treatment will go ahead - add the used equipment
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery'))
-
+ # If consumables, treatment will go ahead
# Log the use of adjuvant chemotherapy
self.get_consumables(
item_codes=self.module.item_codes_breast_can['treatment_chemotherapy'],
@@ -908,8 +904,7 @@ def apply(self, person_id, squeeze_factor):
item_codes=self.module.item_codes_breast_can['palliation'])
if cons_available:
- # If consumables are available and the treatment will go ahead - add the used equipment
- self.add_equipment({'Infusion pump', 'Drip stand'})
+ # If consumables are available and the treatment will go ahead
# Record the start of palliative care if this is first appointment
if pd.isnull(df.at[person_id, "brc_date_palliative_care"]):
diff --git a/src/tlo/methods/cancer_consumables.py b/src/tlo/methods/cancer_consumables.py
index e26d577242..16a6f94f65 100644
--- a/src/tlo/methods/cancer_consumables.py
+++ b/src/tlo/methods/cancer_consumables.py
@@ -3,75 +3,94 @@
"""
from typing import Dict
+from tlo import Module
-def get_consumable_item_codes_cancers(self) -> Dict[str, int]:
+
+def get_consumable_item_codes_cancers(cancer_module: Module) -> Dict[str, int]:
"""
Returns dict the relevant item_codes for the consumables across the five cancer modules. This is intended to prevent
repetition within module code.
"""
- get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name
+ def get_list_of_items(item_list):
+ item_lookup_fn = cancer_module.sim.modules['HealthSystem'].get_item_code_from_item_name
+ return list(map(item_lookup_fn, item_list))
cons_dict = dict()
# Add items that are needed for all cancer modules
- cons_dict['screening_biopsy_endoscopy_cystoscopy_optional'] = \
- {get_item_code("Specimen container"): 1,
- get_item_code("Lidocaine HCl (in dextrose 7.5%), ampoule 2 ml"): 1,
- get_item_code("Gauze, absorbent 90cm x 40m_each_CMST"): 30,
- get_item_code("Disposables gloves, powder free, 100 pieces per box"): 1,
- get_item_code("Syringe, needle + swab"): 1}
+ # todo: @Eva - add syringes, dressing
+ cons_dict['screening_biopsy_core'] = get_list_of_items(['Biopsy needle'])
- cons_dict['screening_biopsy_core'] = \
- {get_item_code("Biopsy needle"): 1}
+ cons_dict['screening_biopsy_optional'] = \
+ get_list_of_items(['Specimen container',
+ 'Lidocaine, injection, 1 % in 20 ml vial',
+ 'Gauze, absorbent 90cm x 40m_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box'])
cons_dict['treatment_surgery_core'] = \
- {get_item_code("Halothane (fluothane)_250ml_CMST"): 100,
- get_item_code("Scalpel blade size 22 (individually wrapped)_100_CMST"): 1}
+ get_list_of_items(['Halothane (fluothane)_250ml_CMST',
+ 'Scalpel blade size 22 (individually wrapped)_100_CMST'])
cons_dict['treatment_surgery_optional'] = \
- {get_item_code("Sodium chloride, injectable solution, 0,9 %, 500 ml"): 2000,
- get_item_code("Paracetamol, tablet, 500 mg"): 8000,
- get_item_code("Pethidine, 50 mg/ml, 2 ml ampoule"): 6,
- get_item_code("Suture pack"): 1,
- get_item_code("Gauze, absorbent 90cm x 40m_each_CMST"): 30,
- get_item_code("Cannula iv (winged with injection pot) 18_each_CMST"): 1}
+ get_list_of_items(['Sodium chloride, injectable solution, 0,9 %, 500 ml',
+ 'Paracetamol, tablet, 500 mg',
+ 'Pethidine, 50 mg/ml, 2 ml ampoule',
+ 'Suture pack',
+ 'Gauze, absorbent 90cm x 40m_each_CMST',
+ 'Cannula iv (winged with injection pot) 18_each_CMST'])
cons_dict['palliation'] = \
- {get_item_code("morphine sulphate 10 mg/ml, 1 ml, injection (nt)_10_IDA"): 1,
- get_item_code("Diazepam, injection, 5 mg/ml, in 2 ml ampoule"): 3,
- get_item_code("Syringe, needle + swab"): 4}
- # N.B. This is not an exhaustive list of drugs required for palliation
-
- cons_dict['treatment_chemotherapy'] = \
- {get_item_code("Cyclophosphamide, 1 g"): 16800}
+ get_list_of_items(['morphine sulphate 10 mg/ml, 1 ml, injection (nt)_10_IDA',
+ 'Diazepam, injection, 5 mg/ml, in 2 ml ampoule',
+ # N.B. This is not an exhaustive list of drugs required for palliation
+ ])
cons_dict['iv_drug_cons'] = \
- {get_item_code("Cannula iv (winged with injection pot) 18_each_CMST"): 1,
- get_item_code("Giving set iv administration + needle 15 drops/ml_each_CMST"): 1,
- get_item_code("Disposables gloves, powder free, 100 pieces per box"): 1,
- get_item_code("Gauze, swabs 8-ply 10cm x 10cm_100_CMST"): 84}
+ get_list_of_items(['Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box'
+ ])
+
+ # Add items that are specific to each cancer module
+ if 'BreastCancer' == cancer_module.name:
+
+ # TODO: chemotharpy protocols??: TAC(Taxotere, Adriamycin, and Cyclophosphamide), AC (anthracycline and
+ # cyclophosphamide) +/-Taxane, TC (Taxotere and cyclophosphamide), CMF (cyclophosphamide, methotrexate,
+ # and fluorouracil), FEC-75 (5-Fluorouracil, Epirubicin, Cyclophosphamide). HER 2 +: Add Trastuzumab
- # Add items that are specific to a particular cancer module
- if 'ProstateCancer' == self.name:
+ # only chemotherapy i consumable list which is also in suggested protocol is cyclo
+ cons_dict['treatment_chemotherapy'] = get_list_of_items(['Cyclophosphamide, 1 g'])
- cons_dict['screening_psa_test_core'] = \
- {get_item_code("Prostate specific antigen test"): 1}
+ elif 'ProstateCancer' == cancer_module.name:
+
+ # TODO: Prostate specific antigen test is listed in ResourceFile_Consumables_availability_and_usage but not
+ # ResourceFile_Consumables_Items_and_Package
+ # cons_dict['screening_psa_test_core'] = get_list_of_items(['Prostate specific antigen test'])
cons_dict['screening_psa_test_optional'] = \
- {get_item_code("Blood collecting tube, 5 ml"): 1,
- get_item_code("Disposables gloves, powder free, 100 pieces per box"): 1,
- get_item_code("Gauze, swabs 8-ply 10cm x 10cm_100_CMST"): 1}
+ get_list_of_items(['Blood collecting tube, 5 ml',
+ 'Disposables gloves, powder free, 100 pieces per box'])
- elif 'BladderCancer' == self.name:
+ elif 'BladderCancer' == cancer_module.name:
# Note: bladder cancer is not in the malawi STG 2023 therefore no details on chemotherapy
- cons_dict['screening_cystoscopy_core'] = \
- {get_item_code("Cystoscope"): 1}
+ # TODO: cytoscope is listed in ResourceFile_Consumables_availability_and_usage but not
+ # ResourceFile_Consumables_Items_and_Packages
+ # cons_dict['screening_cystoscopy_core'] = get_list_of_items(['Cytoscope'])
+
+ cons_dict['screening_cystoscope_optional'] = get_list_of_items(['Specimen container'])
+
+ elif 'OesophagealCancer' == cancer_module.name:
+
+ # TODO: endoscope is listed in ResourceFile_Consumables_availability_and_usage but not
+ # ResourceFile_Consumables_Items_and_Packages
+ # cons_dict['screening_endoscope_core'] = get_list_of_items(['Endoscope'])
- elif 'OesophagealCancer' == self.name:
+ cons_dict['screening_endoscope_optional'] =\
+ get_list_of_items(['Specimen container',
+ 'Gauze, absorbent 90cm x 40m_each_CMST'])
- cons_dict['screening_endoscopy_core'] = \
- {get_item_code("Endoscope"): 1}
+ cons_dict['treatment_chemotherapy'] = get_list_of_items(['Cisplatin 50mg Injection'])
return cons_dict
diff --git a/src/tlo/methods/cardio_metabolic_disorders.py b/src/tlo/methods/cardio_metabolic_disorders.py
index 3c985c2bf1..921b2e0f71 100644
--- a/src/tlo/methods/cardio_metabolic_disorders.py
+++ b/src/tlo/methods/cardio_metabolic_disorders.py
@@ -21,6 +21,7 @@
import pandas as pd
from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata
@@ -28,13 +29,11 @@
from tlo.methods.causes import Cause
from tlo.methods.dxmanager import DxTest
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
from tlo.util import random_date
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
# ---------------------------------------------------------------------------------------------------------
# MODULE DEFINITIONS
@@ -43,7 +42,7 @@
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class CardioMetabolicDisorders(Module, GenericFirstAppointmentsMixin):
+class CardioMetabolicDisorders(Module):
"""
CardioMetabolicDisorders module covers a subset of cardio-metabolic conditions and events. Conditions are binary
and individuals experience a risk of acquiring or losing a condition based on annual probability and
@@ -815,20 +814,19 @@ def on_hsi_alert(self, person_id, treatment_id):
def do_at_generic_first_appt(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs
- ) -> None:
+ ) -> IndividualPropertyUpdates:
# This is called by the HSI generic first appts module whenever a
# person attends an appointment and determines if the person will
# be tested for one or more conditions.
# A maximum of one instance of `HSI_CardioMetabolicDisorders_Investigations`
# is created for the person, during which multiple conditions can
# be investigated.
- if individual_properties["age_years"] <= 5:
- return
+ if patient_details.age_years <= 5:
+ return {}
# The list of conditions that will be investigated in follow-up HSI
conditions_to_investigate = []
@@ -837,11 +835,11 @@ def do_at_generic_first_appt(
# Determine if there are any conditions that should be investigated:
for condition in self.conditions:
- is_already_diagnosed = individual_properties[
- f"nc_{condition}_ever_diagnosed"
- ]
+ is_already_diagnosed = getattr(patient_details, f"nc_{condition}_ever_diagnosed")
has_symptom = f"{condition}_symptoms" in symptoms
- date_of_last_test = individual_properties[f"nc_{condition}_date_last_test"]
+ date_of_last_test = getattr(
+ patient_details, f"nc_{condition}_date_last_test"
+ )
next_test_due = (
pd.isnull(date_of_last_test)
or (self.sim.date - date_of_last_test).days > DAYS_IN_YEAR / 2
@@ -869,32 +867,34 @@ def do_at_generic_first_appt(
if conditions_to_investigate:
event = HSI_CardioMetabolicDisorders_Investigations(
module=self,
- person_id=person_id,
+ person_id=patient_id,
conditions_to_investigate=conditions_to_investigate,
has_any_cmd_symptom=has_any_cmd_symptom,
)
- schedule_hsi_event(event, topen=self.sim.date, priority=0)
+ self.healthsystem.schedule_hsi_event(event, topen=self.sim.date, priority=0)
def do_at_generic_first_appt_emergency(
self,
- person_id: int,
- individual_properties: IndividualProperties,
- symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
+ patient_id: int,
+ patient_details: PatientDetails = None,
+ symptoms: List[str] = None,
**kwargs,
- ) -> None:
- # This is called by the HSI generic first appts module whenever a person attends
- # an emergency appointment and determines if they will receive emergency care
- # based on the duration of time since symptoms have appeared. A maximum of one
- # instance of `HSI_CardioMetabolicDisorders_SeeksEmergencyCareAndGetsTreatment`
- # is created for the person, during which multiple events can be investigated.
+ ) -> IndividualPropertyUpdates:
+ # This is called by the HSI generic first appts module whenever
+ # a person attends an emergency appointment and determines if they
+ # will receive emergency care based on the duration of time since
+ # symptoms have appeared. A maximum of one instance of
+ # `HSI_CardioMetabolicDisorders_SeeksEmergencyCareAndGetsTreatment`
+ # is created for the person, during which multiple events can be
+ # investigated.
ev_to_investigate = []
for ev in self.events:
- # If the person has symptoms of damage from within the last 3 days, schedule
- # them for emergency care
+ # If the person has symptoms of damage from within the last 3 days,
+ # schedule them for emergency care
if f"{ev}_damage" in symptoms and (
(
- self.sim.date - individual_properties[f"nc_{ev}_date_last_event"]
+ self.sim.date
+ - getattr(patient_details, f"nc_{ev}_date_last_event")
).days
<= 3
):
@@ -903,10 +903,10 @@ def do_at_generic_first_appt_emergency(
if ev_to_investigate:
event = HSI_CardioMetabolicDisorders_SeeksEmergencyCareAndGetsTreatment(
module=self,
- person_id=person_id,
+ person_id=patient_id,
events_to_investigate=ev_to_investigate,
)
- schedule_hsi_event(event, topen=self.sim.date, priority=1)
+ self.healthsystem.schedule_hsi_event(event, topen=self.sim.date, priority=1)
class Tracker:
@@ -1306,7 +1306,7 @@ def proportion_of_something_in_a_groupby_ready_for_logging(_df, something, group
df.age_years >= 20)]) / len(df[df[f'nc_{condition}'] & df.is_alive & (df.age_years >= 20)])
}
else:
- diagnosed = {f'{condition}_diagnosis_prevalence': float("nan")}
+ diagnosed = {0.0}
logger.info(
key=f'{condition}_diagnosis_prevalence',
@@ -1320,7 +1320,7 @@ def proportion_of_something_in_a_groupby_ready_for_logging(_df, something, group
df.age_years >= 20)]) / len(df[df[f'nc_{condition}'] & df.is_alive & (df.age_years >= 20)])
}
else:
- on_medication = {f'{condition}_medication_prevalence': float("nan")}
+ on_medication = {0.0}
logger.info(
key=f'{condition}_medication_prevalence',
@@ -1435,7 +1435,6 @@ def apply(self, person_id, squeeze_factor):
return hs.get_blank_appt_footprint()
# Run a test to diagnose whether the person has condition:
- self.add_equipment({'Blood pressure machine'})
dx_result = hs.dx_manager.run_dx_test(
dx_tests_to_run='assess_hypertension',
hsi_event=self
@@ -1488,9 +1487,6 @@ def do_for_each_condition(self, _c) -> bool:
if df.at[person_id, f'nc_{_c}_ever_diagnosed']:
return
- if _c == 'chronic_ischemic_heart_disease':
- self.add_equipment({'Electrocardiogram', 'Stethoscope'})
-
# Run a test to diagnose whether the person has condition:
dx_result = hs.dx_manager.run_dx_test(
dx_tests_to_run=f'assess_{_c}',
@@ -1523,11 +1519,6 @@ def apply(self, person_id, squeeze_factor):
return hs.get_blank_appt_footprint()
# Do test and trigger treatment (if necessary) for each condition:
- if set(self.conditions_to_investigate).intersection(
- ['diabetes', 'chronic_kidney_disease', 'chronic_ischemic_hd']
- ):
- self.add_equipment({'Analyser, Haematology', 'Analyser, Combined Chemistry and Electrolytes'})
-
hsi_scheduled = [self.do_for_each_condition(_c) for _c in self.conditions_to_investigate]
# If no follow-up treatment scheduled but the person has at least 2 risk factors, start weight loss treatment
@@ -1551,7 +1542,6 @@ def apply(self, person_id, squeeze_factor):
and (self.module.rng.rand() < self.module.parameters['hypertension_hsi']['pr_assessed_other_symptoms'])
):
# Run a test to diagnose whether the person has condition:
- self.add_equipment({'Blood pressure machine'})
dx_result = hs.dx_manager.run_dx_test(
dx_tests_to_run='assess_hypertension',
hsi_event=self
@@ -1599,8 +1589,6 @@ def apply(self, person_id, squeeze_factor):
# Don't advise those with CKD to lose weight, but do so for all other conditions if BMI is higher than normal
if self.condition != 'chronic_kidney_disease' and (df.at[person_id, 'li_bmi'] > 2):
- self.add_equipment({'Weighing scale'})
-
self.sim.population.props.at[person_id, 'nc_ever_weight_loss_treatment'] = True
# Schedule a post-weight loss event for individual to potentially lose weight in next 6-12 months:
self.sim.schedule_event(CardioMetabolicDisordersWeightLossEvent(m, person_id),
@@ -1612,23 +1600,10 @@ def apply(self, person_id, squeeze_factor):
return self.sim.modules['HealthSystem'].get_blank_appt_footprint()
assert person[f'nc_{self.condition}_ever_diagnosed'], "The person is not diagnosed and so should not be " \
"receiving an HSI."
-
- # Monthly doses of medications as follows. Diabetes - 1000mg metformin daily (1000*30.5),
- # hypertension - 25mg hydrochlorothiazide daily (25*30.5), CKD 1 dialysis bag (estimate),
- # lower back pain - 2400mg aspirin daily (2400*30.5), CIHD - 75mg aspirin daily (75*30.5)
- dose = {'diabetes': 30_500,
- 'hypertension': 610,
- 'chronic_kidney_disease': 1,
- 'chronic_lower_back_pain': 73_200,
- 'chronic_ischemic_hd': 2288,
- 'ever_stroke': 2288,
- 'ever_heart_attack': 2288}
-
# Check availability of medication for condition
- if self.get_consumables(item_codes=
- {self.module.parameters[f'{self.condition}_hsi'].get(
- 'medication_item_code').astype(int): dose[self.condition]}):
-
+ if self.get_consumables(
+ item_codes=self.module.parameters[f'{self.condition}_hsi'].get('medication_item_code').astype(int)
+ ):
# If medication is available, flag as being on medication
df.at[person_id, f'nc_{self.condition}_on_medication'] = True
# Determine if the medication will work to prevent death
@@ -1694,21 +1669,10 @@ def apply(self, person_id, squeeze_factor):
# Return the blank_appt_footprint() so that this HSI does not occupy any time resources
return self.sim.modules['HealthSystem'].get_blank_appt_footprint()
- # Monthly doses of medications as follows. Diabetes - 1000mg metformin daily (1000*30.5),
- # hypertension - 25mg hydrochlorothiazide daily (25*30.5), CKD 1 dialysis bag (estimate),
- # lower back pain - 2400mg aspirin daily (2400*30.5), CIHD - 75mg aspirin daily (75*30.5)
- dose = {'diabetes': 30_500,
- 'hypertension': 610,
- 'chronic_kidney_disease': 1,
- 'chronic_lower_back_pain': 73_200,
- 'chronic_ischemic_hd': 2288,
- 'ever_stroke': 2288,
- 'ever_heart_attack': 2288}
-
# Check availability of medication for condition
if self.get_consumables(
- item_codes={self.module.parameters[f'{self.condition}_hsi'].get('medication_item_code').astype(int)
- : dose[self.condition]}):
+ item_codes=self.module.parameters[f'{self.condition}_hsi'].get('medication_item_code').astype(int)
+ ):
# Schedule their next HSI for a refill of medication, one month from now
self.sim.modules['HealthSystem'].schedule_hsi_event(
hsi_event=self,
@@ -1761,12 +1725,6 @@ def do_for_each_event_to_be_investigated(self, _ev):
df = self.sim.population.props
# Run a test to diagnose whether the person has condition:
- if _ev == 'ever_stroke':
- self.add_equipment({'Computed Tomography (CT machine)', 'CT scanner accessories'})
-
- if _ev == 'ever_heart_attack':
- self.add_equipment({'Electrocardiogram'})
-
dx_result = self.sim.modules['HealthSystem'].dx_manager.run_dx_test(
dx_tests_to_run=f'assess_{_ev}',
hsi_event=self
@@ -1776,13 +1734,10 @@ def do_for_each_event_to_be_investigated(self, _ev):
df.at[person_id, f'nc_{_ev}_date_diagnosis'] = self.sim.date
df.at[person_id, f'nc_{_ev}_ever_diagnosed'] = True
if self.module.parameters['prob_care_provided_given_seek_emergency_care'] > self.module.rng.random_sample():
-
# If care is provided....
- dose = 20 if _ev == 'ever_stroke' else 40
-
if self.get_consumables(
- item_codes={self.module.parameters[f'{_ev}_hsi'].get(
- 'emergency_medication_item_code').astype(int): dose}
+ item_codes=self.module.parameters[f'{_ev}_hsi'].get(
+ 'emergency_medication_item_code').astype(int)
):
logger.debug(key='debug', data='Treatment will be provided.')
df.at[person_id, f'nc_{_ev}_on_medication'] = True
@@ -1826,7 +1781,6 @@ def apply(self, person_id, squeeze_factor):
data=('This is HSI_CardioMetabolicDisorders_SeeksEmergencyCareAndGetsTreatment: '
f'The squeeze-factor is {squeeze_factor}.'),
)
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ICU'))
for _ev in self.events_to_investigate:
self.do_for_each_event_to_be_investigated(_ev)
diff --git a/src/tlo/methods/care_of_women_during_pregnancy.py b/src/tlo/methods/care_of_women_during_pregnancy.py
index 69ce038299..bc22b86993 100644
--- a/src/tlo/methods/care_of_women_during_pregnancy.py
+++ b/src/tlo/methods/care_of_women_during_pregnancy.py
@@ -1,6 +1,5 @@
from pathlib import Path
-import numpy as np
import pandas as pd
from tlo import DateOffset, Module, Parameter, Property, Types, logging
@@ -203,160 +202,157 @@ def get_and_store_pregnancy_item_codes(self):
This function defines the required consumables for each intervention delivered during this module and stores
them in a module level dictionary called within HSIs
"""
- ic = self.sim.modules['HealthSystem'].get_item_code_from_item_name
+ get_list_of_items = pregnancy_helper_functions.get_list_of_items
- # First we store the item codes for the consumables for which their quantity varies for individuals based on
- # length of pregnancy
# ---------------------------------- BLOOD TEST EQUIPMENT ---------------------------------------------------
self.item_codes_preg_consumables['blood_test_equipment'] = \
- {ic('Blood collecting tube, 5 ml'): 1,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1
- }
+ get_list_of_items(self, ['Blood collecting tube, 5 ml',
+ 'Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box'])
+
# ---------------------------------- IV DRUG ADMIN EQUIPMENT -------------------------------------------------
self.item_codes_preg_consumables['iv_drug_equipment'] = \
- {ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1
- }
+ get_list_of_items(self, ['Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box'])
# -------------------------------------------- ECTOPIC PREGNANCY ---------------------------------------------
self.item_codes_preg_consumables['ectopic_pregnancy_core'] = \
- {ic('Halothane (fluothane)_250ml_CMST'): 100}
+ get_list_of_items(self, ['Halothane (fluothane)_250ml_CMST'])
self.item_codes_preg_consumables['ectopic_pregnancy_optional'] = \
- {ic('Scalpel blade size 22 (individually wrapped)_100_CMST'): 1,
- ic('Sodium chloride, injectable solution, 0,9 %, 500 ml'): 2000,
- ic('Paracetamol, tablet, 500 mg'): 8000,
- ic('Pethidine, 50 mg/ml, 2 ml ampoule'): 6,
- ic('Suture pack'): 1,
- ic('Gauze, absorbent 90cm x 40m_each_CMST'): 30,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1,
- }
+ get_list_of_items(self, ['Scalpel blade size 22 (individually wrapped)_100_CMST',
+ 'Sodium chloride, injectable solution, 0,9 %, 500 ml',
+ 'Paracetamol, tablet, 500 mg',
+ 'Pethidine, 50 mg/ml, 2 ml ampoule',
+ 'Suture pack',
+ 'Gauze, absorbent 90cm x 40m_each_CMST',
+ 'Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box'
+ ])
# ------------------------------------------- POST ABORTION CARE - GENERAL -----------------------------------
self.item_codes_preg_consumables['post_abortion_care_core'] = \
- {ic('Misoprostol, tablet, 200 mcg'): 600}
+ get_list_of_items(self, ['Misoprostol, tablet, 200 mcg'])
self.item_codes_preg_consumables['post_abortion_care_optional'] = \
- {ic('Complete blood count'): 1,
- ic('Blood collecting tube, 5 ml'): 1,
- ic('Paracetamol, tablet, 500 mg'): 8000,
- ic('Gauze, absorbent 90cm x 40m_each_CMST'): 30,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1,
- }
+ get_list_of_items(self, ['Complete blood count',
+ 'Blood collecting tube, 5 ml',
+ 'Paracetamol, tablet, 500 mg',
+ 'Pethidine, 50 mg/ml, 2 ml ampoule',
+ 'Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box'
+ ])
# ------------------------------------------- POST ABORTION CARE - SEPSIS -------------------------------------
self.item_codes_preg_consumables['post_abortion_care_sepsis_core'] = \
- {ic('Benzathine benzylpenicillin, powder for injection, 2.4 million IU'): 8,
- ic('Gentamycin, injection, 40 mg/ml in 2 ml vial'): 6,
- }
+ get_list_of_items(self, ['Benzylpenicillin 3g (5MU), PFR_each_CMST',
+ 'Gentamycin, injection, 40 mg/ml in 2 ml vial'])
self.item_codes_preg_consumables['post_abortion_care_sepsis_optional'] = \
- {ic('Sodium chloride, injectable solution, 0,9 %, 500 ml'): 2000,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1,
- ic('Oxygen, 1000 liters, primarily with oxygen cylinders'): 23_040,
- }
-
- # ------------------------------------------- POST ABORTION CARE - SHOCK ------------------------------------
+ get_list_of_items(self, ['Sodium chloride, injectable solution, 0,9 %, 500 ml',
+ 'Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST',
+ 'Oxygen, 1000 liters, primarily with oxygen cylinders'])
+
+ # ------------------------------------------- POST ABORTION CARE - SHOCK -------------------------------------
self.item_codes_preg_consumables['post_abortion_care_shock'] = \
- {ic('Sodium chloride, injectable solution, 0,9 %, 500 ml'): 2000,
- ic('Oxygen, 1000 liters, primarily with oxygen cylinders'): 23_040,
- }
+ get_list_of_items(self, ['Sodium chloride, injectable solution, 0,9 %, 500 ml',
+ 'Oxygen, 1000 liters, primarily with oxygen cylinders'])
self.item_codes_preg_consumables['post_abortion_care_shock_optional'] = \
- {ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1,
- }
+ get_list_of_items(self, ['Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST'])
+
# ---------------------------------- URINE DIPSTICK ----------------------------------------------------------
- self.item_codes_preg_consumables['urine_dipstick'] = {ic('Urine analysis'): 1}
+ self.item_codes_preg_consumables['urine_dipstick'] = get_list_of_items(self, ['Urine analysis'])
# ---------------------------------- IRON AND FOLIC ACID ------------------------------------------------------
- # Dose changes at run time
- self.item_codes_preg_consumables['iron_folic_acid'] = \
- {ic('Ferrous Salt + Folic Acid, tablet, 200 + 0.25 mg'): 1} # TODO: update con requested here
+ self.item_codes_preg_consumables['iron_folic_acid'] = get_list_of_items(
+ self, ['Ferrous Salt + Folic Acid, tablet, 200 + 0.25 mg'])
# --------------------------------- BALANCED ENERGY AND PROTEIN ----------------------------------------------
- # Dose changes at run time
- self.item_codes_preg_consumables['balanced_energy_protein'] = \
- {ic('Dietary supplements (country-specific)'): 1}
+ self.item_codes_preg_consumables['balanced_energy_protein'] = get_list_of_items(
+ self, ['Dietary supplements (country-specific)'])
# --------------------------------- INSECTICIDE TREATED NETS ------------------------------------------------
- self.item_codes_preg_consumables['itn'] = {ic('Insecticide-treated net'): 1}
+ self.item_codes_preg_consumables['itn'] = get_list_of_items(self, ['Insecticide-treated net'])
# --------------------------------- CALCIUM SUPPLEMENTS -----------------------------------------------------
- self.item_codes_preg_consumables['calcium'] = {ic('Calcium, tablet, 600 mg'): 1}
+ self.item_codes_preg_consumables['calcium'] = get_list_of_items(self, ['Calcium, tablet, 600 mg'])
# -------------------------------- HAEMOGLOBIN TESTING -------------------------------------------------------
- self.item_codes_preg_consumables['hb_test'] = {ic('Haemoglobin test (HB)'): 1}
+ self.item_codes_preg_consumables['hb_test'] = get_list_of_items(self, ['Haemoglobin test (HB)'])
# ------------------------------------------- ALBENDAZOLE -----------------------------------------------------
- self.item_codes_preg_consumables['albendazole'] = {ic('Albendazole 200mg_1000_CMST'): 400}
+ self.item_codes_preg_consumables['albendazole'] = get_list_of_items(self, ['Albendazole 200mg_1000_CMST'])
# ------------------------------------------- HEP B TESTING ---------------------------------------------------
- self.item_codes_preg_consumables['hep_b_test'] = {ic('Hepatitis B test kit-Dertemine_100 tests_CMST'): 1}
+ self.item_codes_preg_consumables['hep_b_test'] = get_list_of_items(
+ self, ['Hepatitis B test kit-Dertemine_100 tests_CMST'])
# ------------------------------------------- SYPHILIS TESTING ------------------------------------------------
- self.item_codes_preg_consumables['syphilis_test'] = {ic('Test, Rapid plasma reagin (RPR)'): 1}
+ self.item_codes_preg_consumables['syphilis_test'] = get_list_of_items(
+ self, ['Test, Rapid plasma reagin (RPR)'])
# ------------------------------------------- SYPHILIS TREATMENT ----------------------------------------------
- self.item_codes_preg_consumables['syphilis_treatment'] =\
- {ic('Benzathine benzylpenicillin, powder for injection, 2.4 million IU'): 1}
+ self.item_codes_preg_consumables['syphilis_treatment'] = get_list_of_items(
+ self, ['Benzathine benzylpenicillin, powder for injection, 2.4 million IU'])
+
+ # ----------------------------------------------- IPTP --------------------------------------------------------
+ self.item_codes_preg_consumables['iptp'] = get_list_of_items(
+ self, ['Sulfamethoxazole + trimethropin, tablet 400 mg + 80 mg'])
# ----------------------------------------------- GDM TEST ----------------------------------------------------
- self.item_codes_preg_consumables['gdm_test'] = {ic('Blood glucose level test'): 1}
+ self.item_codes_preg_consumables['gdm_test'] = get_list_of_items(self, ['Blood glucose level test'])
# ------------------------------------------ FULL BLOOD COUNT -------------------------------------------------
- self.item_codes_preg_consumables['full_blood_count'] = {ic('Complete blood count'): 1}
+ self.item_codes_preg_consumables['full_blood_count'] = get_list_of_items(self, ['Complete blood count'])
# ---------------------------------------- BLOOD TRANSFUSION -------------------------------------------------
- self.item_codes_preg_consumables['blood_transfusion'] = {ic('Blood, one unit'): 2}
+ self.item_codes_preg_consumables['blood_transfusion'] = get_list_of_items(self, ['Blood, one unit'])
# --------------------------------------- ORAL ANTIHYPERTENSIVES ---------------------------------------------
- # Dose changes at run time
- self.item_codes_preg_consumables['oral_antihypertensives'] = {ic('Methyldopa 250mg_1000_CMST'): 1}
+ self.item_codes_preg_consumables['oral_antihypertensives'] = get_list_of_items(
+ self, ['Methyldopa 250mg_1000_CMST'])
# ------------------------------------- INTRAVENOUS ANTIHYPERTENSIVES ---------------------------------------
- self.item_codes_preg_consumables['iv_antihypertensives'] = \
- {ic('Hydralazine, powder for injection, 20 mg ampoule'): 1}
+ self.item_codes_preg_consumables['iv_antihypertensives'] = get_list_of_items(
+ self, ['Hydralazine, powder for injection, 20 mg ampoule'])
# ---------------------------------------- MAGNESIUM SULPHATE ------------------------------------------------
- self.item_codes_preg_consumables['magnesium_sulfate'] = \
- {ic('Magnesium sulfate, injection, 500 mg/ml in 10-ml ampoule'): 2}
+ self.item_codes_preg_consumables['magnesium_sulfate'] = get_list_of_items(
+ self, ['Magnesium sulfate, injection, 500 mg/ml in 10-ml ampoule'])
# ---------------------------------------- MANAGEMENT OF ECLAMPSIA --------------------------------------------
- self.item_codes_preg_consumables['eclampsia_management_optional'] = \
- {ic('Sodium chloride, injectable solution, 0,9 %, 500 ml'): 2000,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1,
- ic('Oxygen, 1000 liters, primarily with oxygen cylinders'): 23_040,
- ic('Complete blood count'): 1,
- ic('Blood collecting tube, 5 ml'): 1,
- ic('Foley catheter'): 1,
- ic('Bag, urine, collecting, 2000 ml'): 1,
- }
+ self.item_codes_preg_consumables['eclampsia_management_optional'] = get_list_of_items(
+ self, ['Misoprostol, tablet, 200 mcg',
+ 'Oxytocin, injection, 10 IU in 1 ml ampoule',
+ 'Sodium chloride, injectable solution, 0,9 %, 500 ml',
+ 'Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box',
+ 'Oxygen, 1000 liters, primarily with oxygen cylinders',
+ 'Complete blood count',
+ 'Blood collecting tube, 5 ml',
+ 'Foley catheter',
+ 'Bag, urine, collecting, 2000 ml'])
# -------------------------------------- ANTIBIOTICS FOR PROM ------------------------------------------------
- self.item_codes_preg_consumables['abx_for_prom'] = \
- {ic('Benzathine benzylpenicillin, powder for injection, 2.4 million IU'): 8}
+ self.item_codes_preg_consumables['abx_for_prom'] = get_list_of_items(
+ self, ['Benzathine benzylpenicillin, powder for injection, 2.4 million IU'])
# ----------------------------------- ORAL DIABETIC MANAGEMENT -----------------------------------------------
- # Dose changes at run time
- self.item_codes_preg_consumables['oral_diabetic_treatment'] = \
- {ic('Glibenclamide 5mg_1000_CMST'): 1}
+ self.item_codes_preg_consumables['oral_diabetic_treatment'] = get_list_of_items(
+ self, ['Glibenclamide 5mg_1000_CMST'])
# ---------------------------------------- INSULIN ----------------------------------------------------------
- # Dose changes at run time
- self.item_codes_preg_consumables['insulin_treatment'] = \
- {ic('Insulin soluble 100 IU/ml, 10ml_each_CMST'): 1}
+ self.item_codes_preg_consumables['insulin_treatment'] = get_list_of_items(
+ self, ['Insulin soluble 100 IU/ml, 10ml_each_CMST'])
def initialise_simulation(self, sim):
@@ -490,9 +486,9 @@ def further_on_birth_care_of_women_in_pregnancy(self, mother_id):
# We log the total number of ANC contacts a woman has undergone at the time of birth via this dictionary
if 'ga_anc_one' in mni[mother_id]:
- ga_anc_one = float(mni[mother_id]['ga_anc_one'])
+ ga_anc_one = mni[mother_id]['ga_anc_one']
else:
- ga_anc_one = 0.0
+ ga_anc_one = 0
total_anc_visit_count = {'person_id': mother_id,
'total_anc': df.at[mother_id, 'ac_total_anc_visits_current_pregnancy'],
@@ -734,7 +730,7 @@ def screening_interventions_delivered_at_every_contact(self, hsi_event):
# check consumables
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event, cons=self.item_codes_preg_consumables['urine_dipstick'], opt_cons=None)
+ self, hsi_event, self.item_codes_preg_consumables, core='urine_dipstick')
# If the intervention will be delivered the dx_manager runs, returning True if the consumables are
# available and the test detects protein in the urine
@@ -747,7 +743,6 @@ def screening_interventions_delivered_at_every_contact(self, hsi_event):
# The process is repeated for blood pressure monitoring
if self.rng.random_sample() < params['prob_intervention_delivered_bp']:
- hsi_event.add_equipment({'Sphygmomanometer'})
if self.sim.modules['HealthSystem'].dx_manager.run_dx_test(dx_tests_to_run='blood_pressure_measurement',
hsi_event=hsi_event):
@@ -793,10 +788,8 @@ def iron_and_folic_acid_supplementation(self, hsi_event):
# check consumable availability - dose is total days of pregnancy x 2 tablets
days = self.get_approx_days_of_pregnancy(person_id)
- updated_cons = {k: v*(days*2) for (k, v) in self.item_codes_preg_consumables['iron_folic_acid'].items()}
-
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event, cons=updated_cons, opt_cons=None)
+ self, hsi_event, self.item_codes_preg_consumables, core='iron_folic_acid', number=days*3)
if avail:
logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'iron_folic_acid'})
@@ -829,11 +822,8 @@ def balance_energy_and_protein_supplementation(self, hsi_event):
# If the consumables are available...
days = self.get_approx_days_of_pregnancy(person_id)
- updated_cons = {k: v*days for (k, v) in
- self.item_codes_preg_consumables['balanced_energy_protein'].items()}
-
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event, cons=updated_cons, opt_cons=None)
+ self, hsi_event, self.item_codes_preg_consumables, core='balanced_energy_protein', number=days)
# And she is deemed to be at risk (i.e. BMI < 18) she is started on supplements
if avail and (df.at[person_id, 'li_bmi'] == 1):
@@ -895,11 +885,8 @@ def calcium_supplementation(self, hsi_event):
or (df.at[person_id, 'la_parity'] > 4)):
days = self.get_approx_days_of_pregnancy(person_id) * 3
- updated_cons = {k: v * days for (k, v) in
- self.item_codes_preg_consumables['calcium'].items()}
-
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event, cons=updated_cons, opt_cons=None)
+ self, hsi_event, self.item_codes_preg_consumables, core='calcium', number=days)
if avail:
df.at[person_id, 'ac_receiving_calcium_supplements'] = True
@@ -922,9 +909,7 @@ def point_of_care_hb_testing(self, hsi_event):
# Run check against probability of testing being delivered
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_preg_consumables['hb_test'],
- opt_cons=self.item_codes_preg_consumables['blood_test_equipment'])
+ self, hsi_event, self.item_codes_preg_consumables, core='hb_test', optional='blood_test_equipment')
# We run the test through the dx_manager and if a woman has anaemia and its detected she will be admitted
# for further care
@@ -998,9 +983,8 @@ def syphilis_screening_and_treatment(self, hsi_event):
logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'syphilis_test'})
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_preg_consumables['syphilis_test'],
- opt_cons=self.item_codes_preg_consumables['blood_test_equipment'])
+ self, hsi_event, self.item_codes_preg_consumables, core='syphilis_test',
+ optional='blood_test_equipment')
test = self.sim.modules['HealthSystem'].dx_manager.run_dx_test(
dx_tests_to_run='blood_test_syphilis', hsi_event=hsi_event)
@@ -1009,9 +993,8 @@ def syphilis_screening_and_treatment(self, hsi_event):
if avail and test:
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_preg_consumables['syphilis_treatment'],
- opt_cons=self.item_codes_preg_consumables['blood_test_equipment'])
+ self, hsi_event, self.item_codes_preg_consumables, core='syphilis_treatment',
+ optional='blood_test_equipment')
if avail:
# We assume that treatment is 100% effective at curing infection
@@ -1076,30 +1059,24 @@ def gdm_screening(self, hsi_event):
if self.rng.random_sample() < params['prob_intervention_delivered_gdm_test']:
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_preg_consumables['gdm_test'],
- opt_cons=self.item_codes_preg_consumables['blood_test_equipment'])
+ self, hsi_event, self.item_codes_preg_consumables, core='gdm_test', optional='blood_test_equipment')
# If the test accurately detects a woman has gestational diabetes the consumables are recorded and
# she is referred for treatment
- if avail:
- hsi_event.add_equipment({'Glucometer'})
+ if avail and self.sim.modules['HealthSystem'].dx_manager.run_dx_test(
+ dx_tests_to_run='blood_test_glucose', hsi_event=hsi_event):
- if (
- self.sim.modules['HealthSystem'].dx_manager.run_dx_test(
- dx_tests_to_run='blood_test_glucose', hsi_event=hsi_event)
- ):
- logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'gdm_screen'})
- mni[person_id]['anc_ints'].append('gdm_screen')
+ logger.info(key='anc_interventions', data={'mother': person_id, 'intervention': 'gdm_screen'})
+ mni[person_id]['anc_ints'].append('gdm_screen')
- # We assume women with a positive GDM screen will be admitted (if they are not already receiving
- # outpatient care)
- if df.at[person_id, 'ac_gest_diab_on_treatment'] == 'none':
+ # We assume women with a positive GDM screen will be admitted (if they are not already receiving
+ # outpatient care)
+ if df.at[person_id, 'ac_gest_diab_on_treatment'] == 'none':
- # Store onset after diagnosis as daly weight is tied to diagnosis
- pregnancy_helper_functions.store_dalys_in_mni(person_id, mni, 'gest_diab_onset',
- self.sim.date)
- df.at[person_id, 'ac_to_be_admitted'] = True
+ # Store onset after diagnosis as daly weight is tied to diagnosis
+ pregnancy_helper_functions.store_dalys_in_mni(person_id, mni, 'gest_diab_onset',
+ self.sim.date)
+ df.at[person_id, 'ac_to_be_admitted'] = True
def interventions_delivered_each_visit_from_anc2(self, hsi_event):
"""This function contains a collection of interventions that are delivered to women every time they attend ANC
@@ -1218,7 +1195,6 @@ def full_blood_count_testing(self, hsi_event):
# If a woman is not truly anaemic but the FBC returns a result of anaemia, due to tests specificity, we
# assume the reported anaemia is mild
hsi_event.get_consumables(item_codes=self.item_codes_preg_consumables['blood_test_equipment'])
- hsi_event.add_equipment({'Analyser, Haematology'})
test_result = self.sim.modules['HealthSystem'].dx_manager.run_dx_test(
dx_tests_to_run='full_blood_count_hb', hsi_event=hsi_event)
@@ -1249,9 +1225,8 @@ def antenatal_blood_transfusion(self, individual_id, hsi_event):
# Check for consumables
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_preg_consumables['blood_transfusion'],
- opt_cons=self.item_codes_preg_consumables['iv_drug_equipment'])
+ self, hsi_event, self.item_codes_preg_consumables, core='blood_transfusion', number=2,
+ optional='iv_drug_equipment')
sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'],
sf='blood_tran',
@@ -1261,8 +1236,6 @@ def antenatal_blood_transfusion(self, individual_id, hsi_event):
if avail and sf_check:
pregnancy_helper_functions.log_met_need(self, 'blood_tran', hsi_event)
- hsi_event.add_equipment({'Drip stand', 'Infusion pump'})
-
# If the woman is receiving blood due to anaemia we apply a probability that a transfusion of 2 units
# RBCs will correct this woman's severe anaemia
if params['treatment_effect_blood_transfusion_anaemia'] > self.rng.random_sample():
@@ -1280,12 +1253,9 @@ def initiate_maintenance_anti_hypertensive_treatment(self, individual_id, hsi_ev
df = self.sim.population.props
# Calculate the approximate dose for the remainder of pregnancy and check availability
- days = self.get_approx_days_of_pregnancy(individual_id) * 4
- updated_cons = {k: v * days for (k, v) in
- self.item_codes_preg_consumables['oral_antihypertensives'].items()}
-
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event, cons=updated_cons, opt_cons=None)
+ self, hsi_event, self.item_codes_preg_consumables, core='oral_antihypertensives',
+ number=(self.get_approx_days_of_pregnancy(individual_id) * 4))
# If the consumables are available then the woman is started on treatment
if avail:
@@ -1304,14 +1274,12 @@ def initiate_treatment_for_severe_hypertension(self, individual_id, hsi_event):
# Define the consumables and check their availability
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_preg_consumables['iv_antihypertensives'],
- opt_cons=self.item_codes_preg_consumables['iv_drug_equipment'])
+ self, hsi_event, self.item_codes_preg_consumables, core='iv_antihypertensives',
+ optional='iv_drug_equipment')
# If they are available then the woman is started on treatment
if avail:
pregnancy_helper_functions.log_met_need(self, 'iv_htns', hsi_event)
- hsi_event.add_equipment({'Drip stand', 'Infusion pump'})
# We assume women treated with antihypertensives would no longer be severely hypertensive- meaning they
# are not at risk of death from severe gestational hypertension in the PregnancySupervisor event
@@ -1337,9 +1305,8 @@ def treatment_for_severe_pre_eclampsia_or_eclampsia(self, individual_id, hsi_eve
df = self.sim.population.props
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_preg_consumables['magnesium_sulfate'],
- opt_cons=self.item_codes_preg_consumables['eclampsia_management_optional'])
+ self, hsi_event, self.item_codes_preg_consumables, core='magnesium_sulfate',
+ optional='eclampsia_management_optional')
# check HCW will deliver intervention
sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'],
@@ -1350,7 +1317,6 @@ def treatment_for_severe_pre_eclampsia_or_eclampsia(self, individual_id, hsi_eve
if avail and sf_check:
df.at[individual_id, 'ac_mag_sulph_treatment'] = True
pregnancy_helper_functions.log_met_need(self, 'mag_sulph', hsi_event)
- hsi_event.add_equipment({'Drip stand', 'Infusion pump'})
def antibiotics_for_prom(self, individual_id, hsi_event):
"""
@@ -1363,9 +1329,8 @@ def antibiotics_for_prom(self, individual_id, hsi_event):
# check consumables and whether HCW are available to deliver the intervention
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_preg_consumables['abx_for_prom'],
- opt_cons=self.item_codes_preg_consumables['iv_drug_equipment'])
+ self, hsi_event, self.item_codes_preg_consumables, core='abx_for_prom',
+ optional='iv_drug_equipment')
sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'],
sf='iv_abx',
@@ -1373,7 +1338,6 @@ def antibiotics_for_prom(self, individual_id, hsi_event):
if avail and sf_check:
df.at[individual_id, 'ac_received_abx_for_prom'] = True
- hsi_event.add_equipment({'Drip stand', 'Infusion pump'})
def ectopic_pregnancy_treatment_doesnt_run(self, hsi_event):
"""
@@ -1463,12 +1427,6 @@ def apply(self, person_id, squeeze_factor):
df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1
# =================================== INTERVENTIONS ====================================================
- # Add equipment used during first ANC visit not directly related to interventions
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC'))
- self.add_equipment(
- {'Height Pole (Stadiometer)', 'MUAC tape',
- 'Ultrasound, combined 2/4 pole interferential with vacuum and dual frequency 1-3MHZ'})
-
# First all women, regardless of ANC contact or gestation, undergo urine and blood pressure measurement
# and depression screening
self.module.screening_interventions_delivered_at_every_contact(hsi_event=self)
@@ -1487,7 +1445,6 @@ def apply(self, person_id, squeeze_factor):
# If the woman presents after 20 weeks she is provided interventions she has missed by presenting late
if mother.ps_gestational_age_in_weeks > 19:
- self.add_equipment({'Stethoscope, foetal, monaural, Pinard, plastic'})
self.module.point_of_care_hb_testing(hsi_event=self)
self.module.albendazole_administration(hsi_event=self)
self.module.iptp_administration(hsi_event=self)
@@ -1552,12 +1509,7 @@ def apply(self, person_id, squeeze_factor):
df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1
# =================================== INTERVENTIONS ====================================================
- # Add equipment used during ANC visit not directly related to interventions
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC'))
- self.add_equipment(
- {'Ultrasound, combined 2/4 pole interferential with vacuum and dual frequency 1-3MHZ'})
-
- # First we administer the interventions all women will receive at this contact regardless of
+ # First we administer the administer the interventions all women will receive at this contact regardless of
# gestational age
self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self)
self.module.tetanus_vaccination(hsi_event=self)
@@ -1641,8 +1593,6 @@ def apply(self, person_id, squeeze_factor):
df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1
# =================================== INTERVENTIONS ====================================================
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC'))
-
gest_age_next_contact = self.module.determine_gestational_age_for_next_contact(person_id)
self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self)
@@ -1715,8 +1665,6 @@ def apply(self, person_id, squeeze_factor):
df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1
# =================================== INTERVENTIONS ====================================================
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC'))
-
gest_age_next_contact = self.module.determine_gestational_age_for_next_contact(person_id)
self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self)
@@ -1784,11 +1732,7 @@ def apply(self, person_id, squeeze_factor):
self.module.anc_counter[5] += 1
df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1
- # =================================== INTERVENTIONS ===================================================
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC'))
- self.add_equipment(
- {'Ultrasound, combined 2/4 pole interferential with vacuum and dual frequency 1-3MHZ'})
-
+ # =================================== INTERVENTIONS ====================================================
gest_age_next_contact = self.module.determine_gestational_age_for_next_contact(person_id)
self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self)
@@ -1856,9 +1800,6 @@ def apply(self, person_id, squeeze_factor):
gest_age_next_contact = self.module.determine_gestational_age_for_next_contact(person_id)
# =================================== INTERVENTIONS ====================================================
- self.add_equipment({'Weighing scale', 'Measuring tapes',
- 'Stethoscope, foetal, monaural, Pinard, plastic'})
-
self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self)
if mother.ps_gestational_age_in_weeks < 40:
@@ -1918,8 +1859,6 @@ def apply(self, person_id, squeeze_factor):
df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1
# =================================== INTERVENTIONS ====================================================
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC'))
-
gest_age_next_contact = self.module.determine_gestational_age_for_next_contact(person_id)
self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self)
@@ -1973,8 +1912,6 @@ def apply(self, person_id, squeeze_factor):
self.module.anc_counter[8] += 1
df.at[person_id, 'ac_total_anc_visits_current_pregnancy'] += 1
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ANC'))
-
self.module.interventions_delivered_each_visit_from_anc2(hsi_event=self)
if df.at[person_id, 'ac_to_be_admitted']:
@@ -2489,8 +2426,6 @@ def apply(self, person_id, squeeze_factor):
if not mother.la_currently_in_labour and not mother.hs_is_inpatient and mother.ps_gest_diab != 'none' \
and (mother.ac_gest_diab_on_treatment != 'none') and (mother.ps_gestational_age_in_weeks > 21):
- est_length_preg = self.module.get_approx_days_of_pregnancy(person_id)
-
def schedule_gdm_event_and_checkup():
# Schedule GestationalDiabetesGlycaemicControlEvent which determines if this new treatment will
# effectively control blood glucose prior to next follow up
@@ -2515,12 +2450,9 @@ def schedule_gdm_event_and_checkup():
# meds
if mother.ac_gest_diab_on_treatment == 'diet_exercise':
- days = est_length_preg * 10
- updated_cons = {k: v * days for (k, v) in
- self.module.item_codes_preg_consumables['oral_diabetic_treatment'].items()}
-
avail = pregnancy_helper_functions.return_cons_avail(
- self.module, self, cons=updated_cons, opt_cons=None)
+ self.module, self, self.module.item_codes_preg_consumables, core='oral_diabetic_treatment',
+ number=(self.module.get_approx_days_of_pregnancy(person_id) * 2))
# If the meds are available women are started on that treatment
if avail:
@@ -2536,15 +2468,9 @@ def schedule_gdm_event_and_checkup():
# blood sugar- they are started on insulin
if mother.ac_gest_diab_on_treatment == 'orals':
- # Dose is (avg.) 0.8 units per KG per day. Average weight is an appoximation
- required_units_per_preg = 65 * (0.8 * est_length_preg)
- required_vials = np.ceil(required_units_per_preg/1000)
-
- updated_cons = {k: v * required_vials for (k, v) in
- self.module.item_codes_preg_consumables['insulin_treatment'].items()}
-
avail = pregnancy_helper_functions.return_cons_avail(
- self.module, self, cons=updated_cons, opt_cons=None)
+ self.module, self, self.module.item_codes_preg_consumables, core='insulin_treatment',
+ number=5)
if avail:
df.at[person_id, 'ac_gest_diab_on_treatment'] = 'insulin'
@@ -2588,51 +2514,43 @@ def apply(self, person_id, squeeze_factor):
# Request baseline PAC consumables
baseline_cons = pregnancy_helper_functions.return_cons_avail(
- self.module, self,
- cons=self.module.item_codes_preg_consumables['post_abortion_care_core'],
- opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_optional'])
+ self.module, self, self.module.item_codes_preg_consumables, core='post_abortion_care_core',
+ optional='post_abortion_care_optional')
# Check HCW availability to deliver surgical removal of retained products
sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.sim.modules['Labour'],
sf='retained_prod',
hsi_event=self)
- # Add used equipment if intervention can happen
- if baseline_cons and sf_check:
- self.add_equipment({'D&C set', 'Suction Curettage machine', 'Drip stand', 'Infusion pump'})
-
# Then we determine if a woman gets treatment for her complication depending on availability of the baseline
# consumables (misoprostol) or a HCW who can conduct MVA/DC (we dont model equipment) and additional
# consumables for management of her specific complication
if abortion_complications.has_any([person_id], 'sepsis', first=True):
cons_for_sepsis_pac = pregnancy_helper_functions.return_cons_avail(
- self.module, self,
- cons=self.module.item_codes_preg_consumables['post_abortion_care_sepsis_core'],
- opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_sepsis_optional'])
+ self.module, self, self.module.item_codes_preg_consumables, core='post_abortion_care_sepsis_core',
+ optional='post_abortion_care_sepsis_optional')
if cons_for_sepsis_pac and (baseline_cons or sf_check):
df.at[person_id, 'ac_received_post_abortion_care'] = True
elif abortion_complications.has_any([person_id], 'haemorrhage', first=True):
+
cons_for_haemorrhage = pregnancy_helper_functions.return_cons_avail(
- self.module, self,
- cons=self.module.item_codes_preg_consumables['blood_transfusion'],
- opt_cons=self.module.item_codes_preg_consumables['iv_drug_equipment'])
+ self.module, self, self.module.item_codes_preg_consumables, core='blood_transfusion', number=2,
+ optional='iv_drug_equipment')
cons_for_shock = pregnancy_helper_functions.return_cons_avail(
- self.module, self,
- cons=self.module.item_codes_preg_consumables['post_abortion_care_shock'],
- opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_shock_optional'])
+ self.module, self, self.module.item_codes_preg_consumables, core='post_abortion_care_shock',
+ optional='post_abortion_care_shock_optional')
if cons_for_haemorrhage and cons_for_shock and (baseline_cons or sf_check):
df.at[person_id, 'ac_received_post_abortion_care'] = True
elif abortion_complications.has_any([person_id], 'injury', first=True):
cons_for_shock = pregnancy_helper_functions.return_cons_avail(
- self.module, self,
- cons=self.module.item_codes_preg_consumables['post_abortion_care_shock'],
- opt_cons=self.module.item_codes_preg_consumables['post_abortion_care_shock_optional'])
+ self.module, self, self.module.item_codes_preg_consumables, core='post_abortion_care_shock',
+ optional='post_abortion_care_shock_optional')
if cons_for_shock and (baseline_cons or sf_check):
df.at[person_id, 'ac_received_post_abortion_care'] = True
@@ -2677,15 +2595,13 @@ def apply(self, person_id, squeeze_factor):
# We define the required consumables and check their availability
avail = pregnancy_helper_functions.return_cons_avail(
- self.module, self,
- cons=self.module.item_codes_preg_consumables['ectopic_pregnancy_core'],
- opt_cons=self.module.item_codes_preg_consumables['ectopic_pregnancy_optional'])
+ self.module, self, self.module.item_codes_preg_consumables, core='ectopic_pregnancy_core',
+ optional='ectopic_pregnancy_optional')
# If they are available then treatment can go ahead
if avail:
self.sim.modules['PregnancySupervisor'].mother_and_newborn_info[person_id]['delete_mni'] = True
pregnancy_helper_functions.log_met_need(self.module, 'ep_case_mang', self)
- self.add_equipment({'Laparotomy Set'})
# For women who have sought care after they have experienced rupture we use this treatment variable to
# reduce risk of death (women who present prior to rupture do not pass through the death event as we assume
diff --git a/src/tlo/methods/chronicsyndrome.py b/src/tlo/methods/chronicsyndrome.py
index 0ae6599939..8d466149d8 100644
--- a/src/tlo/methods/chronicsyndrome.py
+++ b/src/tlo/methods/chronicsyndrome.py
@@ -1,27 +1,22 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, List
+from typing import List
import numpy as np
import pandas as pd
from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.methods import Metadata
from tlo.methods.causes import Cause
from tlo.methods.demography import InstantaneousDeath
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
-if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
-
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class ChronicSyndrome(Module, GenericFirstAppointmentsMixin):
+class ChronicSyndrome(Module):
"""
This is a dummy chronic disease
It demonstrates the following behaviours in respect of the healthsystem module:
@@ -285,18 +280,17 @@ def report_daly_values(self):
def do_at_generic_first_appt_emergency(
self,
- person_id: int,
+ patient_id: int,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
"""Example for CHRONIC SYNDROME"""
if "craving_sandwiches" in symptoms:
event = HSI_ChronicSyndrome_SeeksEmergencyCareAndGetsTreatment(
module=self,
- person_id=person_id,
+ person_id=patient_id,
)
- schedule_hsi_event(event, topen=self.sim.date, priority=1)
+ self.healthsystem.schedule_hsi_event(event, topen=self.sim.date, priority=1)
class ChronicSyndromeEvent(RegularEvent, PopulationScopeEventMixin):
diff --git a/src/tlo/methods/contraception.py b/src/tlo/methods/contraception.py
index ab6c633f4c..67d6684fce 100644
--- a/src/tlo/methods/contraception.py
+++ b/src/tlo/methods/contraception.py
@@ -646,60 +646,23 @@ def select_contraceptive_following_birth(self, mother_id, mother_age):
def get_item_code_for_each_contraceptive(self):
"""Get the item_code for each contraceptive and for contraceptive initiation."""
+ # TODO: update with optional items (currently all considered essential)
- # ### Get item codes from item names and define number of units per case here
- get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name
+ get_items_from_pkg = self.sim.modules['HealthSystem'].get_item_codes_from_package_name
_cons_codes = dict()
- # # items for each method that requires an HSI to switch to
- # in 80% cases combined pills administrated
- # in other 20% cases same amount of progesterone-only pills ("Levonorgestrel 0.0375 mg, cycle") administrated
- # (omitted in here)
- _cons_codes['pill'] = \
- {get_item_code("Levonorgestrel 0.15 mg + Ethinyl estradiol 30 mcg (Microgynon), cycle"): 21 * 3.75}
- _cons_codes['male_condom'] =\
- {get_item_code("Condom, male"): 30}
- _cons_codes['other_modern'] =\
- {get_item_code("Female Condom_Each_CMST"): 30}
- _cons_codes['IUD'] =\
- {get_item_code("Glove disposable powdered latex medium_100_CMST"): 2,
- get_item_code("IUD, Copper T-380A"): 1}
- _cons_codes['injections'] = \
- {get_item_code("Depot-Medroxyprogesterone Acetate 150 mg - 3 monthly"): 1,
- get_item_code("Glove disposable powdered latex medium_100_CMST"): 1,
- get_item_code("Water for injection, 10ml_Each_CMST"): 1,
- get_item_code("Povidone iodine, solution, 10 %, 5 ml per injection"): 5,
- get_item_code("Gauze, swabs 8-ply 10cm x 10cm_100_CMST"): 1}
- _cons_codes['implant'] =\
- {get_item_code("Glove disposable powdered latex medium_100_CMST"): 3,
- get_item_code("Lidocaine HCl (in dextrose 7.5%), ampoule 2 ml"): 2,
- get_item_code("Povidone iodine, solution, 10 %, 5 ml per injection"): 1*5, # unit: 1 ml
- get_item_code("Syringe, needle + swab"): 2,
- get_item_code("Trocar"): 1,
- get_item_code("Needle suture intestinal round bodied ½ circle trocar_6_CMST"): 1,
- # in 50% cases Jadelle administrated
- # in other 50% cases other type of implant ("Implanon (Etonogestrel 68 mg)") administrated
- # (omitted in here)
- get_item_code("Jadelle (implant), box of 2_CMST"): 1,
- get_item_code("Gauze, swabs 8-ply 10cm x 10cm_100_CMST"): 1}
- _cons_codes['female_sterilization'] =\
- {get_item_code("Lidocaine HCl (in dextrose 7.5%), ampoule 2 ml"): 1,
- get_item_code("Atropine sulphate 600 micrograms/ml, 1ml_each_CMST"): 0.5, # 1 unit used only in 50% cases
- # approximated by 0.5 unit each time
- get_item_code("Diazepam, injection, 5 mg/ml, in 2 ml ampoule"): 1,
- get_item_code("Syringe, autodestruct, 5ml, disposable, hypoluer with 21g needle_each_CMST"): 3,
- get_item_code("Gauze, swabs 8-ply 10cm x 10cm_100_CMST"): 2,
- get_item_code("Needle, suture, assorted sizes, round body"): 3,
- get_item_code("Suture, catgut, chromic, 0, 150 cm"): 3,
- get_item_code("Tape, adhesive, 2.5 cm wide, zinc oxide, 5 m roll"): 125, # unit: 1 cm long (2.5 cm wide)
- get_item_code("Glove surgeon's size 7 sterile_2_CMST"): 2,
- get_item_code("Paracetamol, tablet, 500 mg"): 8*500, # unit: 1 mg
- get_item_code("Povidone iodine, solution, 10 %, 5 ml per injection"): 2*5, # unit: 1 ml
- get_item_code("Cotton wool, 500g_1_CMST"): 100} # unit: 1 g
-
+ # items for each method that requires an HSI to switch to
+ _cons_codes['pill'] = get_items_from_pkg('Pill')
+ _cons_codes['male_condom'] = get_items_from_pkg('Male condom')
+ _cons_codes['other_modern'] = get_items_from_pkg('Female Condom')
+ # NB. The consumable female condom is used for the contraceptive state of "other_modern method"
+ _cons_codes['IUD'] = get_items_from_pkg('IUD')
+ _cons_codes['injections'] = get_items_from_pkg('Injectable')
+ _cons_codes['implant'] = get_items_from_pkg('Implant')
+ _cons_codes['female_sterilization'] = get_items_from_pkg('Female sterilization')
assert set(_cons_codes.keys()) == set(self.states_that_may_require_HSI_to_switch_to)
# items used when initiating a modern reliable method after not using or switching from non-reliable method
- _cons_codes['co_initiation'] = {get_item_code('Pregnancy slide test kit_100_CMST'): 1}
+ _cons_codes['co_initiation'] = get_items_from_pkg('Contraception initiation')
return _cons_codes
@@ -1181,12 +1144,8 @@ def apply(self, person_id, squeeze_factor):
# Record the date that Family Planning Appointment happened for this person
self.sim.population.props.at[person_id, "co_date_of_last_fp_appt"] = self.sim.date
- # Measure weight, height and BP even if contraception not administrated
- self.add_equipment({
- 'Weighing scale', 'Height Pole (Stadiometer)', 'Blood pressure machine'
- })
-
# Determine essential and optional items
+ # TODO: we don't distinguish essential X optional for contraception methods yet, will need to update once we do
items_essential = self.module.cons_codes[self.new_contraceptive]
items_optional = {}
# Record use of consumables and default the person to "not_using" if the consumable is not available.
@@ -1211,8 +1170,7 @@ def apply(self, person_id, squeeze_factor):
items_all = {**items_essential, **items_optional}
# Determine whether the contraception is administrated (ie all essential items are available),
- # if so do log the availability of all items and update used equipment if any, if not set the contraception to
- # "not_using":
+ # if so do log the availability of all items, if not set the contraception to "not_using":
co_administrated = all(v for k, v in cons_available.items() if k in items_essential)
if co_administrated:
@@ -1236,18 +1194,6 @@ def apply(self, person_id, squeeze_factor):
)
_new_contraceptive = self.new_contraceptive
-
- # Add used equipment
- if _new_contraceptive == 'female_sterilization':
- self.add_equipment({
- 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Lamp, Anglepoise'
- })
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Minor Surgery'))
- elif _new_contraceptive == 'IUD':
- self.add_equipment({
- 'Cusco’s/ bivalved Speculum (small, medium, large)', 'Sponge Holding Forceps'
- })
-
else:
_new_contraceptive = "not_using"
diff --git a/src/tlo/methods/copd.py b/src/tlo/methods/copd.py
index 53602505ae..7e85e57ee2 100644
--- a/src/tlo/methods/copd.py
+++ b/src/tlo/methods/copd.py
@@ -7,18 +7,17 @@
from tlo import Module, Parameter, Property, Types, logging
from tlo.analysis.utils import flatten_multi_index_series_into_dict_for_logging
+from tlo.core import ConsumablesChecker, IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata
from tlo.methods.causes import Cause
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
from tlo.util import random_date
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import ConsumablesChecker, HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
@@ -30,11 +29,11 @@
}
-class Copd(Module, GenericFirstAppointmentsMixin):
+class Copd(Module):
"""The module responsible for determining Chronic Obstructive Pulmonary Diseases (COPD) status and outcomes.
and initialises parameters and properties associated with COPD plus functions and events related to COPD."""
- INIT_DEPENDENCIES = {'SymptomManager', 'Lifestyle', 'HealthSystem'}
+ INIT_DEPENDENCIES = {'SymptomManager', 'Lifestyle'}
ADDITIONAL_DEPENDENCIES = set()
METADATA = {
@@ -191,14 +190,14 @@ def define_symptoms(self):
def lookup_item_codes(self):
"""Look-up the item-codes for the consumables needed in the HSI Events for this module."""
- ic = self.sim.modules['HealthSystem'].get_item_code_from_item_name
-
+ # todo: Need to look-up these item-codes.
self.item_codes = {
- 'bronchodilater_inhaler': ic('Salbutamol Inhaler 100mcg/dose - 200 doses '),
- 'oxygen': ic('Oxygen, 1000 liters, primarily with oxygen cylinders'),
- 'aminophylline': ic('Aminophylline 100mg, tablets'),
- 'amoxycillin': ic('Amoxycillin 250mg_1000_CMST'),
- 'prednisolone': ic('Prednisolone 5mg_100_CMST'),
+ 'bronchodilater_inhaler': 293,
+ 'steroid_inhaler': 294,
+ 'oxygen': 127,
+ 'aminophylline': 292,
+ 'amoxycillin': 125,
+ 'prednisolone': 291
}
def do_logging(self):
@@ -213,10 +212,9 @@ def do_logging(self):
def _common_first_appt(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
consumables_checker: ConsumablesChecker,
):
"""What to do when a person presents at the generic first appt HSI
@@ -225,53 +223,52 @@ def _common_first_appt(
* Otherwise --> just give inhaler.
"""
if ('breathless_moderate' in symptoms) or ('breathless_severe' in symptoms):
+ patient_details_updates = {}
# Give inhaler if patient does not already have one
- if not individual_properties["ch_has_inhaler"] and consumables_checker(
- {self.item_codes["bronchodilater_inhaler"]: 1}
+ if not patient_details.ch_has_inhaler and consumables_checker(
+ self.item_codes["bronchodilater_inhaler"]
):
- individual_properties["ch_has_inhaler"] = True
+ patient_details_updates["ch_has_inhaler"] = True
+
if "breathless_severe" in symptoms:
event = HSI_Copd_TreatmentOnSevereExacerbation(
- module=self, person_id=person_id
+ module=self, person_id=patient_id
)
- schedule_hsi_event(
+ self.healthsystem.schedule_hsi_event(
event, topen=self.sim.date, priority=0
)
+ return patient_details_updates
def do_at_generic_first_appt(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
consumables_checker: ConsumablesChecker,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
# Non-emergency appointments are only forwarded if
# the patient is over 5 years old
- if individual_properties["age_years"] > 5:
+ if patient_details.age_years > 5:
return self._common_first_appt(
- person_id=person_id,
- individual_properties=individual_properties,
+ patient_id=patient_id,
+ patient_details=patient_details,
symptoms=symptoms,
- schedule_hsi_event=schedule_hsi_event,
consumables_checker=consumables_checker,
)
def do_at_generic_first_appt_emergency(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
consumables_checker: ConsumablesChecker,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
return self._common_first_appt(
- person_id=person_id,
- individual_properties=individual_properties,
+ patient_id=patient_id,
+ patient_details=patient_details,
symptoms=symptoms,
- schedule_hsi_event=schedule_hsi_event,
consumables_checker=consumables_checker,
)
@@ -574,8 +571,7 @@ def apply(self, person_id, squeeze_factor):
* Provide treatment: whatever is available at this facility at this time (no referral).
"""
df = self.sim.population.props
- # Assume average 8L O2 for 2 days inpatient care
- if not self.get_consumables({self.module.item_codes['oxygen']: 23_040}):
+ if not self.get_consumables(self.module.item_codes['oxygen']):
# refer to the next higher facility if the current facility has no oxygen
self.facility_levels_index += 1
if self.facility_levels_index >= len(self.all_facility_levels):
@@ -585,13 +581,11 @@ def apply(self, person_id, squeeze_factor):
else:
# Give oxygen and AminoPhylline, if possible, ... and cancel death if the treatment is successful.
- # Aminophylline dose = 100mg 8hrly, assuming 600mg in 48 hours
prob_treatment_success = self.module.models.prob_livesaved_given_treatment(
df=df.iloc[[person_id]],
- oxygen=self.get_consumables({self.module.item_codes['oxygen']: 23_040}),
- aminophylline=self.get_consumables({self.module.item_codes['aminophylline']: 600})
+ oxygen=self.get_consumables(self.module.item_codes['oxygen']),
+ aminophylline=self.get_consumables(self.module.item_codes['aminophylline'])
)
- self.add_equipment({'Oxygen cylinder, with regulator', 'Nasal Prongs', 'Drip stand', 'Infusion pump'})
if prob_treatment_success:
df.at[person_id, 'ch_will_die_this_episode'] = False
diff --git a/src/tlo/methods/demography.py b/src/tlo/methods/demography.py
index e58f3895f4..f3d3e3a8c4 100644
--- a/src/tlo/methods/demography.py
+++ b/src/tlo/methods/demography.py
@@ -26,7 +26,6 @@
logging,
)
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
-from tlo.logging.helpers import get_dataframe_row_as_dict_for_logging
from tlo.methods.causes import (
Cause,
collect_causes_from_disease_modules,
@@ -125,6 +124,7 @@ def __init__(self, name=None, resourcefilepath=None, equal_allocation_by_distric
'date_of_death': Property(Types.DATE, 'Date of death of this individual'),
'sex': Property(Types.CATEGORICAL, 'Male or female', categories=['M', 'F']),
'mother_id': Property(Types.INT, 'Unique identifier of mother of this individual'),
+ 'district_num_of_residence': Property(Types.INT, 'The district number in which the person is resident'),
# the categories of these properties are set in `pre_initialise_population`
'cause_of_death': Property(
@@ -133,12 +133,6 @@ def __init__(self, name=None, resourcefilepath=None, equal_allocation_by_distric
categories=['SET_AT_RUNTIME']
),
- 'district_num_of_residence': Property(
- Types.CATEGORICAL,
- 'The district number in which the person is resident',
- categories=['SET_AT_RUNTIME']
- ),
-
'district_of_residence': Property(
Types.CATEGORICAL,
'The district (name) of residence (mapped from district_num_of_residence).',
@@ -226,11 +220,6 @@ def pre_initialise_population(self):
'The cause of death of this individual (the tlo_cause defined by the module)',
categories=list(self.causes_of_death.keys())
)
- self.PROPERTIES['district_num_of_residence'] = Property(
- Types.CATEGORICAL,
- 'The district (name) of residence (mapped from district_num_of_residence).',
- categories=sorted(self.parameters['district_num_to_region_name']),
- )
self.PROPERTIES['district_of_residence'] = Property(
Types.CATEGORICAL,
'The district (name) of residence (mapped from district_num_of_residence).',
@@ -508,7 +497,7 @@ def do_death(self, individual_id: int, cause: str, originating_module: Module):
data_to_log_for_each_death = {
'age': person['age_years'],
'sex': person['sex'],
- 'cause': str(cause),
+ 'cause': cause,
'label': self.causes_of_death[cause].label,
'person_id': individual_id,
'li_wealth': person['li_wealth'] if 'li_wealth' in person else -99,
@@ -524,7 +513,7 @@ def do_death(self, individual_id: int, cause: str, originating_module: Module):
# - log all the properties for the deceased person
logger_detail.info(key='properties_of_deceased_persons',
- data=get_dataframe_row_as_dict_for_logging(df, individual_id),
+ data=person.to_dict(),
description='values of all properties at the time of death for deceased persons')
# - log the death in the Deviance module (if it is registered)
@@ -645,11 +634,11 @@ def apply(self, population):
df = population.props
dates_of_birth = df.loc[df.is_alive, 'date_of_birth']
df.loc[df.is_alive, 'age_exact_years'] = age_at_date(
- self.module.sim.date, dates_of_birth
+ population.sim.date, dates_of_birth
)
df.loc[df.is_alive, 'age_years'] = df.loc[df.is_alive, 'age_exact_years'].astype('int64')
df.loc[df.is_alive, 'age_range'] = df.loc[df.is_alive, 'age_years'].map(self.age_range_lookup)
- df.loc[df.is_alive, 'age_days'] = (self.module.sim.date - dates_of_birth).dt.days
+ df.loc[df.is_alive, 'age_days'] = (population.sim.date - dates_of_birth).dt.days
class OtherDeathPoll(RegularEvent, PopulationScopeEventMixin):
@@ -810,7 +799,7 @@ def apply(self, population):
num_children = pd.Series(index=range(5), data=0).add(
df[df.is_alive & (df.age_years < 5)].groupby('age_years').size(),
fill_value=0
- ).astype(int)
+ )
logger.info(key='num_children', data=num_children.to_dict())
diff --git a/src/tlo/methods/depression.py b/src/tlo/methods/depression.py
index a0ffdd12b2..c7f2577382 100644
--- a/src/tlo/methods/depression.py
+++ b/src/tlo/methods/depression.py
@@ -4,24 +4,23 @@
from __future__ import annotations
from pathlib import Path
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, List, Optional
import numpy as np
import pandas as pd
from tlo import DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import DiagnosisFunction, IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata
from tlo.methods.causes import Cause
from tlo.methods.dxmanager import DxTest
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import DiagnosisFunction, HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
@@ -31,7 +30,7 @@
# MODULE DEFINITIONS
# ---------------------------------------------------------------------------------------------------------
-class Depression(Module, GenericFirstAppointmentsMixin):
+class Depression(Module):
def __init__(self, name=None, resourcefilepath=None):
super().__init__(name)
self.resourcefilepath = resourcefilepath
@@ -593,30 +592,24 @@ def do_on_presentation_to_care(self, person_id: int, hsi_event: HSI_Event):
and there may need to be screening for depression.
"""
if self._check_for_suspected_depression(
- self.sim.modules["SymptomManager"].has_what(person_id=person_id),
+ self.sim.modules["SymptomManager"].has_what(person_id),
hsi_event.TREATMENT_ID,
self.sim.population.props.at[person_id, "de_ever_diagnosed_depression"],
):
- individual_properties = {}
- self.do_when_suspected_depression(
- person_id=person_id,
- individual_properties=individual_properties,
- schedule_hsi_event=self.sim.modules["HealthSystem"].schedule_hsi_event,
- hsi_event=hsi_event
+ patient_details_updates = self.do_when_suspected_depression(
+ person_id=person_id, hsi_event=hsi_event
)
- self.sim.population.props.loc[person_id, individual_properties.keys()] = (
- individual_properties.values()
+ self.sim.population.props.loc[person_id, patient_details_updates.keys()] = (
+ patient_details_updates.values()
)
return
def do_when_suspected_depression(
self,
person_id: int,
- individual_properties: Union[dict, IndividualProperties],
- schedule_hsi_event: HSIEventScheduler,
diagnosis_function: Optional[DiagnosisFunction] = None,
hsi_event: Optional[HSI_Event] = None,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
"""
This is called by any HSI event when depression is suspected or otherwise investigated.
@@ -628,11 +621,12 @@ def do_when_suspected_depression(
runs diagnosis tests.
:param person_id: Patient's row index in the population DataFrame.
- :param individual_properties: Indexable object to write individual property updates to.
- :param schedule_hsi_event: Function to schedule subsequent HSI events.
:param diagnosis_function: A function capable of running diagnosis checks on the population.
:param hsi_event: The HSI_Event that triggered this call.
+ :returns: Values as per the output of do_at_generic_first_appt().
"""
+ patient_details_updates = {}
+
if diagnosis_function is None:
assert isinstance(
hsi_event, HSI_Event
@@ -649,51 +643,49 @@ def diagnosis_function(tests, use_dict: bool = False, report_tried: bool = False
# Assess for depression and initiate treatments for depression if positive diagnosis
if diagnosis_function('assess_depression'):
# If depressed: diagnose the person with depression
- individual_properties['de_ever_diagnosed_depression'] = True
+ patient_details_updates['de_ever_diagnosed_depression'] = True
scheduling_options = {"priority": 0, "topen": self.sim.date}
# Provide talking therapy
# (this can occur even if the person has already had talking therapy before)
- schedule_hsi_event(
+ self.healthsystem.schedule_hsi_event(
HSI_Depression_TalkingTherapy(module=self, person_id=person_id),
**scheduling_options,
)
# Initiate person on anti-depressants
# (at the same facility level as the HSI event that is calling)
- schedule_hsi_event(
+ self.healthsystem.schedule_hsi_event(
HSI_Depression_Start_Antidepressant(module=self, person_id=person_id),
**scheduling_options,
)
+ return patient_details_updates
def do_at_generic_first_appt(
- self, individual_properties: IndividualProperties, **kwargs
- ) -> None:
- if individual_properties["age_years"] > 5:
- self.do_at_generic_first_appt_emergency(
- individual_properties=individual_properties,
+ self,
+ patient_details: PatientDetails,
+ **kwargs
+ ) -> IndividualPropertyUpdates:
+ if patient_details.age_years > 5:
+ return self.do_at_generic_first_appt_emergency(
+ patient_details=patient_details,
**kwargs,
)
def do_at_generic_first_appt_emergency(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
diagnosis_function: DiagnosisFunction,
treatment_id: str,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
if self._check_for_suspected_depression(
- symptoms,
- treatment_id,
- individual_properties["de_ever_diagnosed_depression"],
+ symptoms, treatment_id, patient_details.de_ever_diagnosed_depression
):
- self.do_when_suspected_depression(
- person_id=person_id,
- individual_properties=individual_properties,
+ return self.do_when_suspected_depression(
+ person_id=patient_id,
diagnosis_function=diagnosis_function,
- schedule_hsi_event=schedule_hsi_event,
)
@@ -869,10 +861,10 @@ def apply(self, population):
n_ever_talk_ther = (df.de_ever_talk_ther & df.is_alive & df.de_depr).sum()
def zero_out_nan(x):
- return x if not np.isnan(x) else 0.0
+ return x if not np.isnan(x) else 0
def safe_divide(x, y):
- return float(x / y) if y > 0.0 else 0.0
+ return x / y if y > 0.0 else 0.0
dict_for_output = {
'prop_ge15_depr': zero_out_nan(safe_divide(n_ge15_depr, n_ge15)),
@@ -961,10 +953,9 @@ def apply(self, person_id, squeeze_factor):
"receiving an HSI. "
# Check availability of antidepressant medication
- # Dose is 25mg daily, patient provided with month supply - 25mg x 30.437 (days) = 761mg per month
- item_code_with_dose = {self.module.parameters['anti_depressant_medication_item_code']: 761}
+ item_code = self.module.parameters['anti_depressant_medication_item_code']
- if self.get_consumables(item_codes=item_code_with_dose):
+ if self.get_consumables(item_codes=item_code):
# If medication is available, flag as being on antidepressants
df.at[person_id, 'de_on_antidepr'] = True
@@ -1005,10 +996,7 @@ def apply(self, person_id, squeeze_factor):
return self.sim.modules['HealthSystem'].get_blank_appt_footprint()
# Check availability of antidepressant medication
- # Dose is 25mg daily, patient provided with month supply - 25mg x 30.437 (days) = 761mg per month
- item_code_with_dose = {self.module.parameters['anti_depressant_medication_item_code']: 761}
-
- if self.get_consumables(item_codes=item_code_with_dose):
+ if self.get_consumables(self.module.parameters['anti_depressant_medication_item_code']):
# Schedule their next HSI for a refill of medication, one month from now
self.sim.modules['HealthSystem'].schedule_hsi_event(
hsi_event=HSI_Depression_Refill_Antidepressant(person_id=person_id, module=self.module),
diff --git a/src/tlo/methods/diarrhoea.py b/src/tlo/methods/diarrhoea.py
index 06c8a37b18..1d5537bf53 100644
--- a/src/tlo/methods/diarrhoea.py
+++ b/src/tlo/methods/diarrhoea.py
@@ -13,30 +13,30 @@
Outstanding Issues
* To include rotavirus vaccine
+ * See todo
"""
from __future__ import annotations
from collections.abc import Iterable
from pathlib import Path
-from typing import TYPE_CHECKING, List
+from typing import TYPE_CHECKING
import numpy as np
import pandas as pd
from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import DiagnosisFunction, IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata
from tlo.methods.causes import Cause
from tlo.methods.dxmanager import DxTest
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.util import random_date, sample_outcome
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import DiagnosisFunction, HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
@@ -46,7 +46,7 @@
# MODULE DEFINITIONS
# ---------------------------------------------------------------------------------------------------------
-class Diarrhoea(Module, GenericFirstAppointmentsMixin):
+class Diarrhoea(Module):
# Declare the pathogens that this module will simulate:
pathogens = [
'rotavirus',
@@ -651,23 +651,18 @@ def report_daly_values(self):
def look_up_consumables(self):
"""Look up and store the consumables item codes used in each of the HSI."""
- ic = self.sim.modules['HealthSystem'].get_item_code_from_item_name
-
- self.consumables_used_in_hsi['ORS'] = {ic('ORS, sachet'): 2}
-
- self.consumables_used_in_hsi['Treatment_Severe_Dehydration'] = \
- {ic('ORS, sachet'): 2,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 1000}
-
- self.consumables_used_in_hsi['Zinc'] = ic('Zinc, tablet, 20 mg')
-
- # For weight based treatment for children under five, we've averaged the median weight for each for years
- # 0-5 as 12kg.
- # So for cipro/para - 10mg/kg 12 hrly for 7 days = ((10*12)*2) * 7 (same dose in mg reccomended)
- self.consumables_used_in_hsi['Antibiotics_for_Dysentery'] = \
- {ic('Ciprofloxacin 250mg_100_CMST'): 1680,
- ic("Paracetamol syrup 120mg/5ml_0.0119047619047619_CMST"): 70} # 24mg/ml so 1680/24 = 70ml per dose
+ get_item_codes_from_package_name = self.sim.modules['HealthSystem'].get_item_codes_from_package_name
+
+ self.consumables_used_in_hsi['ORS'] = get_item_codes_from_package_name(
+ package='ORS')
+ self.consumables_used_in_hsi['Treatment_Severe_Dehydration'] = get_item_codes_from_package_name(
+ package='Treatment of severe diarrhea')
+ self.consumables_used_in_hsi['Zinc_Under6mo'] = get_item_codes_from_package_name(
+ package='Zinc for Children 0-6 months')
+ self.consumables_used_in_hsi['Zinc_Over6mo'] = get_item_codes_from_package_name(
+ package='Zinc for Children 6-59 months')
+ self.consumables_used_in_hsi['Antibiotics_for_Dysentery'] = get_item_codes_from_package_name(
+ package='Antibiotics for treatment of dysentery')
def do_treatment(self, person_id, hsi_event):
"""Method called by the HSI that enacts decisions about a treatment and its effect for diarrhoea caused by a
@@ -711,10 +706,9 @@ def do_treatment(self, person_id, hsi_event):
# ** Implement the procedure for treatment **
# STEP ZERO: Get the Zinc consumable (happens irrespective of whether child will die or not)
- # Dose is 10mg 24hrly for 10 days <6months or 20m for >6mnths
- dose = 100 if person.age_exact_years < 0.5 else 200
gets_zinc = hsi_event.get_consumables(
- item_codes={self.consumables_used_in_hsi['Zinc']: dose}
+ item_codes=self.consumables_used_in_hsi[
+ 'Zinc_Under6mo' if person.age_exact_years < 0.5 else 'Zinc_Over6mo']
)
# STEP ONE: Aim to alleviate dehydration:
@@ -948,18 +942,16 @@ def check_properties(self):
def do_at_generic_first_appt(
self,
- person_id: int,
- individual_properties: IndividualProperties,
- schedule_hsi_event: HSIEventScheduler,
- symptoms: List[str],
+ patient_id: int,
+ patient_details: PatientDetails,
diagnosis_function: DiagnosisFunction,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
# This routine is called when Diarrhoea is a symptom for a child
# attending a Generic HSI Appointment. It checks for danger signs
# and schedules HSI Events appropriately.
- if individual_properties["age_years"] > 5 or "diarrhoea" not in symptoms:
- return
+ if patient_details.age_years > 5:
+ return {}
# 1) Assessment of danger signs
danger_signs = diagnosis_function(
@@ -972,11 +964,11 @@ def do_at_generic_first_appt(
self.rng.rand() < self.parameters["prob_hospitalization_on_danger_signs"]
)
hsi_event_class = (
- HSI_Diarrhoea_Treatment_Inpatient if is_inpatient else
+ HSI_Diarrhoea_Treatment_Inpatient if is_inpatient else
HSI_Diarrhoea_Treatment_Outpatient
)
- event = hsi_event_class(person_id=person_id, module=self)
- schedule_hsi_event(event, priority=0, topen=self.sim.date)
+ event = hsi_event_class(person_id=patient_id, module=self)
+ self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date)
class Models:
@@ -1571,8 +1563,6 @@ def apply(self, person_id, squeeze_factor):
if not df.at[person_id, 'is_alive']:
return
- self.add_equipment({'Infusion pump', 'Drip stand'})
-
self.module.do_treatment(person_id=person_id, hsi_event=self)
diff --git a/src/tlo/methods/enhanced_lifestyle.py b/src/tlo/methods/enhanced_lifestyle.py
index 26c79d9587..008424ec2b 100644
--- a/src/tlo/methods/enhanced_lifestyle.py
+++ b/src/tlo/methods/enhanced_lifestyle.py
@@ -12,7 +12,6 @@
from tlo.analysis.utils import flatten_multi_index_series_into_dict_for_logging
from tlo.events import PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
-from tlo.logging.helpers import grouped_counts_with_all_combinations
from tlo.util import get_person_id_to_inherit_from
logger = logging.getLogger(__name__)
@@ -1940,42 +1939,33 @@ def apply(self, population):
for _property in all_lm_keys:
if _property in log_by_age_15up:
if _property in cat_by_rural_urban_props:
- data = grouped_counts_with_all_combinations(
- df.loc[df.is_alive & (df.age_years >= 15)],
- ["li_urban", "sex", _property, "age_range"]
- )
+ data = df.loc[df.is_alive & (df.age_years >= 15)].groupby(by=[
+ 'li_urban', 'sex', _property, 'age_range']).size()
else:
- data = grouped_counts_with_all_combinations(
- df.loc[df.is_alive & (df.age_years >= 15)],
- ["sex", _property, "age_range"]
- )
+ data = df.loc[df.is_alive & (df.age_years >= 15)].groupby(by=[
+ 'sex', _property, 'age_range']).size()
+
elif _property == 'li_in_ed':
- data = grouped_counts_with_all_combinations(
- df.loc[df.is_alive & df.age_years.between(5, 19)],
- ["sex", "li_wealth", "li_in_ed", "age_years"],
- {"age_years": range(5, 20)}
- )
+ data = df.loc[df.is_alive & df.age_years.between(5, 19)].groupby(by=[
+ 'sex', 'li_wealth', _property, 'age_years']).size()
+
elif _property == 'li_ed_lev':
- data = grouped_counts_with_all_combinations(
- df.loc[df.is_alive & df.age_years.between(15, 49)],
- ["sex", "li_wealth", "li_ed_lev", "age_years"],
- {"age_years": range(15, 50)}
- )
+ data = df.loc[df.is_alive & df.age_years.between(15, 49)].groupby(by=[
+ 'sex', 'li_wealth', _property, 'age_years']).size()
+
elif _property == 'li_is_sexworker':
- data = grouped_counts_with_all_combinations(
- df.loc[df.is_alive & (df.age_years.between(15, 49))],
- ["sex", "li_is_sexworker", "age_range"],
- )
+ data = df.loc[df.is_alive & (df.age_years.between(15, 49))].groupby(by=[
+ 'sex', _property, 'age_range']).size()
+
elif _property in cat_by_rural_urban_props:
# log all properties that are also categorised by rural or urban in addition to ex and age groups
- data = grouped_counts_with_all_combinations(
- df.loc[df.is_alive], ["li_urban", "sex", _property, "age_range"]
- )
+ data = df.loc[df.is_alive].groupby(by=[
+ 'li_urban', 'sex', _property, 'age_range']).size()
+
else:
# log all other remaining properties
- data = grouped_counts_with_all_combinations(
- df.loc[df.is_alive], ["sex", _property, "age_range"]
- )
+ data = df.loc[df.is_alive].groupby(by=['sex', _property, 'age_range']).size()
+
# log data
logger.info(
key=_property,
diff --git a/src/tlo/methods/epilepsy.py b/src/tlo/methods/epilepsy.py
index cc8c0f8cca..6c4ff0e41d 100644
--- a/src/tlo/methods/epilepsy.py
+++ b/src/tlo/methods/epilepsy.py
@@ -1,28 +1,23 @@
-from __future__ import annotations
-
from pathlib import Path
-from typing import TYPE_CHECKING, List, Union
+from typing import List, Union
import numpy as np
import pandas as pd
from tlo import DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.methods import Metadata
from tlo.methods.causes import Cause
from tlo.methods.demography import InstantaneousDeath
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
-if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
-
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class Epilepsy(Module, GenericFirstAppointmentsMixin):
+class Epilepsy(Module):
def __init__(self, name=None, resourcefilepath=None):
super().__init__(name)
self.resourcefilepath = resourcefilepath
@@ -100,15 +95,6 @@ def __init__(self, name=None, resourcefilepath=None):
'daly_wt_epilepsy_seizure_free': Parameter(
Types.REAL, 'disability weight for less severe epilepsy' 'controlled phase - code 862'
),
- 'prob_start_anti_epilep_when_seizures_detected_in_generic_first_appt': Parameter(
- Types.REAL, 'probability that someone who has had a seizure is started on anti-epileptics. This is '
- 'calibrated to induce the correct proportion of persons with epilepsy currently receiving '
- 'anti-epileptics.'
- ),
- 'max_num_of_failed_attempts_before_defaulting': Parameter(
- Types.INT, 'maximum number of time an HSI can be repeated if the relevant essential consumables are not '
- 'available.'
- ),
}
"""
@@ -409,20 +395,13 @@ def get_best_available_medicine(self, hsi_event) -> Union[None, str]:
def do_at_generic_first_appt_emergency(
self,
- person_id: int,
+ patient_id: int,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
if "seizures" in symptoms:
- # Determine if treatment will start - depends on probability of prescribing, which is calibrated to
- # induce the right proportion of persons with epilepsy receiving treatment.
-
- prob_start = self.parameters['prob_start_anti_epilep_when_seizures_detected_in_generic_first_appt']
-
- if self.rng.random_sample() < prob_start:
- event = HSI_Epilepsy_Start_Anti_Epileptic(person_id=person_id, module=self)
- schedule_hsi_event(event, priority=0, topen=self.sim.date)
+ event = HSI_Epilepsy_Start_Anti_Epileptic(person_id=patient_id, module=self)
+ self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date)
class EpilepsyEvent(RegularEvent, PopulationScopeEventMixin):
@@ -578,30 +557,25 @@ def apply(self, population):
n_seiz_stat_1_3 = sum(status_groups.iloc[1:].is_alive)
n_seiz_stat_2_3 = sum(status_groups.iloc[2:].is_alive)
- n_antiep = int((df.is_alive & df.ep_antiep).sum())
+ n_antiep = (df.is_alive & df.ep_antiep).sum()
- n_epi_death = int(df.ep_epi_death.sum())
+ n_epi_death = df.ep_epi_death.sum()
status_groups['prop_seiz_stats'] = status_groups.is_alive / sum(status_groups.is_alive)
status_groups['prop_seiz_stat_on_anti_ep'] = status_groups['ep_antiep'] / status_groups.is_alive
status_groups['prop_seiz_stat_on_anti_ep'] = status_groups['prop_seiz_stat_on_anti_ep'].fillna(0)
epi_death_rate = \
- (n_epi_death * 4 * 1000) / n_seiz_stat_2_3 if n_seiz_stat_2_3 > 0 else 0.0
+ (n_epi_death * 4 * 1000) / n_seiz_stat_2_3 if n_seiz_stat_2_3 > 0 else 0
cum_deaths = (~df.is_alive).sum()
- # Proportion of those with infrequent or frequent seizures currently on anti-epileptics
- prop_freq_or_infreq_seiz_on_antiep = status_groups[2:].ep_antiep.sum() / status_groups[2:].is_alive.sum() \
- if status_groups[2:].is_alive.sum() > 0 else 0
-
logger.info(key='epilepsy_logging',
data={
'prop_seiz_stat_0': status_groups['prop_seiz_stats'].iloc[0],
'prop_seiz_stat_1': status_groups['prop_seiz_stats'].iloc[1],
'prop_seiz_stat_2': status_groups['prop_seiz_stats'].iloc[2],
'prop_seiz_stat_3': status_groups['prop_seiz_stats'].iloc[3],
- 'prop_freq_or_infreq_seiz_on_antiep': prop_freq_or_infreq_seiz_on_antiep,
'prop_antiepilep_seiz_stat_0': status_groups['prop_seiz_stat_on_anti_ep'].iloc[0],
'prop_antiepilep_seiz_stat_1': status_groups['prop_seiz_stat_on_anti_ep'].iloc[1],
'prop_antiepilep_seiz_stat_2': status_groups['prop_seiz_stat_on_anti_ep'].iloc[2],
@@ -628,9 +602,6 @@ def __init__(self, module, person_id):
self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({'Over5OPD': 1})
self.ACCEPTED_FACILITY_LEVEL = '1b'
- self._MAX_NUMBER_OF_FAILED_ATTEMPTS_BEFORE_DEFAULTING = module.parameters['max_num_of_failed_attempts_before_defaulting']
- self._counter_of_failed_attempts_due_to_unavailable_medicines = 0
-
def apply(self, person_id, squeeze_factor):
df = self.sim.population.props
hs = self.sim.modules["HealthSystem"]
@@ -640,14 +611,7 @@ def apply(self, person_id, squeeze_factor):
if best_available_medicine is not None:
# Request the medicine from the health system
-
- dose = {'phenobarbitone': 9131, # 100mg per day - 3 months
- 'carbamazepine': 91_311, # 1000mg per day - 3 months
- 'phenytoin': 27_393} # 300mg per day - 3 months
-
- self.get_consumables({self.module.item_codes[best_available_medicine]:
- dose[best_available_medicine]})
-
+ self.get_consumables(self.module.item_codes[best_available_medicine])
# Update this person's properties to show that they are currently on medication
df.at[person_id, 'ep_antiep'] = True
@@ -662,12 +626,8 @@ def apply(self, person_id, squeeze_factor):
priority=0
)
- elif (
- self._counter_of_failed_attempts_due_to_unavailable_medicines
- < self._MAX_NUMBER_OF_FAILED_ATTEMPTS_BEFORE_DEFAULTING
- ):
+ else:
# If no medicine is available, run this HSI again next month
- self._counter_of_failed_attempts_due_to_unavailable_medicines += 1
self.module.sim.modules['HealthSystem'].schedule_hsi_event(hsi_event=self,
topen=self.sim.date + pd.DateOffset(months=1),
tclose=None,
@@ -679,7 +639,7 @@ class HSI_Epilepsy_Follow_Up(HSI_Event, IndividualScopeEventMixin):
def __init__(self, module, person_id):
super().__init__(module, person_id=person_id)
- self._MAX_NUMBER_OF_FAILED_ATTEMPTS_BEFORE_DEFAULTING = module.parameters['max_num_of_failed_attempts_before_defaulting']
+ self._MAX_NUMBER_OF_FAILED_ATTEMPTS_BEFORE_DEFAULTING = 2
self._DEFAULT_APPT_FOOTPRINT = self.make_appt_footprint({'Over5OPD': 1})
self._REPEATED_APPT_FOOTPRINT = self.make_appt_footprint({'PharmDispensing': 1})
@@ -702,30 +662,18 @@ def apply(self, person_id, squeeze_factor):
# Request the medicine
best_available_medicine = self.module.get_best_available_medicine(self)
if best_available_medicine is not None:
-
- # Schedule a reoccurrence of this follow-up in 3 months if ep_seiz_stat == '3',
- # else, schedule this reoccurrence of it in 1 year (i.e., if ep_seiz_stat == '2'
- if df.at[person_id, 'ep_seiz_stat'] == '3':
- fu_mnths = 3
- else:
- fu_mnths = 12
-
# The medicine is available, so request it
- dose = {'phenobarbitone_3_mnths': 9131, 'phenobarbitone_12_mnths': 36_525, # 100mg per day - 3/12 months
- 'carbamazepine_3_mnths': 91_311, 'carbamazepine_12_mnths': 365_250, # 1000mg per day - 3/12 months
- 'phenytoin_3_mnths': 27_393, 'phenytoin_12_mnths': 109_575} # 300mg per day - 3/12 months
-
- self.get_consumables({self.module.item_codes[best_available_medicine]:
- dose[f'{best_available_medicine}_{fu_mnths}_mnths']})
+ self.get_consumables(self.module.item_codes[best_available_medicine])
# Reset counter of "failed attempts" and put the appointment for the next occurrence to the usual
self._counter_of_failed_attempts_due_to_unavailable_medicines = 0
self.EXPECTED_APPT_FOOTPRINT = self._DEFAULT_APPT_FOOTPRINT
- # Schedule follow-up
+ # Schedule a reoccurrence of this follow-up in 3 months if ep_seiz_stat == '3',
+ # else, schedule this reoccurrence of it in 1 year (i.e., if ep_seiz_stat == '2')
hs.schedule_hsi_event(
hsi_event=self,
- topen=self.sim.date + DateOffset(months=fu_mnths),
+ topen=self.sim.date + DateOffset(months=3 if df.at[person_id, 'ep_seiz_stat'] == '3' else 12),
tclose=None,
priority=0
)
diff --git a/src/tlo/methods/equipment.py b/src/tlo/methods/equipment.py
index 62776fb3ad..916927d792 100644
--- a/src/tlo/methods/equipment.py
+++ b/src/tlo/methods/equipment.py
@@ -1,12 +1,11 @@
import warnings
from collections import defaultdict
-from typing import Counter, Dict, Iterable, Literal, Set, Union
+from typing import Counter, Iterable, Literal, Set, Union
import numpy as np
import pandas as pd
from tlo import logging
-from tlo.logging.helpers import get_dataframe_row_as_dict_for_logging
logger_summary = logging.getLogger("tlo.methods.healthsystem.summary")
@@ -78,7 +77,6 @@ def __init__(
# - Data structures for quick look-ups for items and descriptors
self._item_code_lookup = self.catalogue.set_index('Item_Description')['Item_Code'].to_dict()
- self._pkg_lookup = self._create_pkg_lookup()
self._all_item_descriptors = set(self._item_code_lookup.keys())
self._all_item_codes = set(self._item_code_lookup.values())
self._all_fac_ids = self.master_facilities_list['Facility_ID'].unique()
@@ -117,7 +115,7 @@ def _get_equipment_availability_probabilities(self) -> pd.Series:
calculation if the equipment availability change event occurs during the simulation.
"""
dat = self.data_availability.set_index(
- [self.data_availability["Facility_ID"].astype(np.int64), self.data_availability["Item_Code"].astype(np.int64)]
+ [self.data_availability["Facility_ID"].astype(int), self.data_availability["Item_Code"].astype(int)]
)["Pr_Available"]
# Confirm that there is an estimate for every item_code at every facility_id
@@ -136,11 +134,11 @@ def parse_items(self, items: Union[int, str, Iterable[int], Iterable[str]]) -> S
def check_item_codes_recognised(item_codes: set[int]):
if not item_codes.issubset(self._all_item_codes):
- warnings.warn(f'At least one item code was unrecognised: "{item_codes}".')
+ warnings.warn(f'Item code(s) "{item_codes}" not recognised.')
def check_item_descriptors_recognised(item_descriptors: set[str]):
if not item_descriptors.issubset(self._all_item_descriptors):
- warnings.warn(f'At least one item descriptor was unrecognised "{item_descriptors}".')
+ warnings.warn(f'Item descriptor(s) "{item_descriptors}" not recognised.')
# Make into a set if it is not one already
if isinstance(items, (str, int)):
@@ -221,16 +219,16 @@ def write_to_log(self) -> None:
mfl = self.master_facilities_list
- def sorted_keys_or_empty_list(x: Union[dict, None]) -> list:
- if isinstance(x, dict):
- return sorted(x.keys())
+ def set_of_keys_or_empty_set(x: Union[set, dict]):
+ if isinstance(x, set):
+ return x
+ elif isinstance(x, dict):
+ return set(x.keys())
else:
- return []
+ return set()
set_of_equipment_ever_used_at_each_facility_id = pd.Series({
- fac_id: sorted_keys_or_empty_list(
- self._record_of_equipment_used_by_facility_id.get(fac_id)
- )
+ fac_id: set_of_keys_or_empty_set(self._record_of_equipment_used_by_facility_id.get(fac_id, set()))
for fac_id in mfl['Facility_ID']
}, name='EquipmentEverUsed').astype(str)
@@ -240,50 +238,23 @@ def sorted_keys_or_empty_list(x: Union[dict, None]) -> list:
right_index=True,
how='left',
).drop(columns=['Facility_ID', 'Facility_Name'])
+
# Log multi-row data-frame
- for row_index in output.index:
+ for _, row in output.iterrows():
logger_summary.info(
key='EquipmentEverUsed_ByFacilityID',
description='For each facility_id (the set of facilities of the same level in a district), the set of'
'equipment items that are ever used.',
- data=get_dataframe_row_as_dict_for_logging(output, row_index)
+ data=row.to_dict(),
)
- def from_pkg_names(self, pkg_names: Union[str, Iterable[str]]) -> Set[int]:
- """Convenience function to find the set of item_codes that are grouped under requested package name(s) in the
- catalogue."""
- # Make into a set if it is not one already
- if isinstance(pkg_names, (str, int)):
- pkg_names = set([pkg_names])
- else:
- pkg_names = set(pkg_names)
-
- item_codes = set()
- for pkg_name in pkg_names:
- if pkg_name in self._pkg_lookup.keys():
- item_codes.update(self._pkg_lookup[pkg_name])
- else:
- raise ValueError(f'That Pkg_Name is not in the catalogue: {pkg_name=}')
-
- return item_codes
-
- def _create_pkg_lookup(self) -> Dict[str, Set[int]]:
- """Create a lookup from a Package Name to a set of Item_Codes that are contained with that package.
- N.B. In the Catalogue, there is one row for each Item, and the Packages to which each Item belongs (if any)
- is given in a column 'Pkg_Name': if an item belongs to multiple packages, these names are separated by commas,
- and if it doesn't belong to any package, then there is a NULL value."""
+ def lookup_item_codes_from_pkg_name(self, pkg_name: str) -> Set[int]:
+ """Convenience function to find the set of item_codes that are grouped under a package name in the catalogue.
+ It is expected that this is used by the disease module once and then the resulting equipment item_codes are
+ saved on the module."""
df = self.catalogue
- # Make dataframe with columns for each package, and bools showing whether each item_code is included
- pkgs = df['Pkg_Name'].replace({float('nan'): None}) \
- .str.get_dummies(sep=',') \
- .set_index(df.Item_Code) \
- .astype(bool)
-
- # Make dict of the form: {'Pkg_Code': }
- pkg_lookup_dict = {
- pkg_name.strip(): set(pkgs[pkg_name].loc[pkgs[pkg_name]].index.to_list())
- for pkg_name in pkgs.columns
- }
+ if pkg_name not in df['Pkg_Name'].unique():
+ raise ValueError(f'That Pkg_Name is not in the catalogue: {pkg_name=}')
- return pkg_lookup_dict
+ return set(df.loc[df['Pkg_Name'] == pkg_name, 'Item_Code'].values)
diff --git a/src/tlo/methods/healthseekingbehaviour.py b/src/tlo/methods/healthseekingbehaviour.py
index 22e628d166..f29dc37fcc 100644
--- a/src/tlo/methods/healthseekingbehaviour.py
+++ b/src/tlo/methods/healthseekingbehaviour.py
@@ -5,28 +5,23 @@
The write-up of these estimates is: Health-seeking behaviour estimates for adults and children.docx
"""
-from __future__ import annotations
-
from pathlib import Path
-from typing import TYPE_CHECKING, List
+from typing import List
import numpy as np
import pandas as pd
from tlo import Date, DateOffset, Module, Parameter, Types
+from tlo.core import IndividualPropertyUpdates
from tlo.events import PopulationScopeEventMixin, Priority, RegularEvent
from tlo.lm import LinearModel
from tlo.methods import Metadata
from tlo.methods.hsi_generic_first_appts import (
- GenericFirstAppointmentsMixin,
HSI_EmergencyCare_SpuriousSymptom,
HSI_GenericEmergencyFirstAppt,
HSI_GenericNonEmergencyFirstAppt,
)
-if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
-
# ---------------------------------------------------------------------------------------------------------
# MODULE DEFINITIONS
# ---------------------------------------------------------------------------------------------------------
@@ -34,7 +29,7 @@
HIGH_ODDS_RATIO = 1e5
-class HealthSeekingBehaviour(Module, GenericFirstAppointmentsMixin):
+class HealthSeekingBehaviour(Module):
"""
This modules determines if the onset of symptoms will lead to that person presenting at the health
facility for a HSI_GenericFirstAppointment.
@@ -260,17 +255,16 @@ def force_any_symptom_to_lead_to_healthcareseeking(self):
def do_at_generic_first_appt_emergency(
self,
- person_id: int,
+ patient_id: int,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
if "spurious_emergency_symptom" in symptoms:
event = HSI_EmergencyCare_SpuriousSymptom(
module=self.sim.modules["HealthSeekingBehaviour"],
- person_id=person_id,
+ person_id=patient_id,
)
- schedule_hsi_event(event, priority=0, topen=self.sim.date)
+ self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date)
# ---------------------------------------------------------------------------------------------------------
# REGULAR POLLING EVENT
diff --git a/src/tlo/methods/healthsystem.py b/src/tlo/methods/healthsystem.py
index 5c6b2022e1..1dbd47fb63 100644
--- a/src/tlo/methods/healthsystem.py
+++ b/src/tlo/methods/healthsystem.py
@@ -961,7 +961,7 @@ def format_daily_capabilities(self, use_funded_or_actual_staffing: str) -> tuple
self.parameters[f'Daily_Capabilities_{use_funded_or_actual_staffing}']
)
capabilities = capabilities.rename(columns={'Officer_Category': 'Officer_Type_Code'}) # neaten
-
+
# Create new column where capabilities per staff are computed
capabilities['Mins_Per_Day_Per_Staff'] = capabilities['Total_Mins_Per_Day']/capabilities['Staff_Count']
@@ -984,10 +984,10 @@ def format_daily_capabilities(self, use_funded_or_actual_staffing: str) -> tuple
# Merge in information about facility from Master Facilities List
mfl = self.parameters['Master_Facilities_List']
capabilities_ex = capabilities_ex.merge(mfl, on='Facility_ID', how='left')
-
+
# Create a copy of this to store staff counts
capabilities_per_staff_ex = capabilities_ex.copy()
-
+
# Merge in information about officers
# officer_types = self.parameters['Officer_Types_Table'][['Officer_Type_Code', 'Officer_Type']]
# capabilities_ex = capabilities_ex.merge(officer_types, on='Officer_Type_Code', how='left')
@@ -1000,7 +1000,7 @@ def format_daily_capabilities(self, use_funded_or_actual_staffing: str) -> tuple
how='left',
)
capabilities_ex = capabilities_ex.fillna(0)
-
+
capabilities_per_staff_ex = capabilities_per_staff_ex.merge(
capabilities[['Facility_ID', 'Officer_Type_Code', 'Mins_Per_Day_Per_Staff']],
on=['Facility_ID', 'Officer_Type_Code'],
@@ -1015,7 +1015,7 @@ def format_daily_capabilities(self, use_funded_or_actual_staffing: str) -> tuple
+ '_Officer_'
+ capabilities_ex['Officer_Type_Code']
)
-
+
# Give the standard index:
capabilities_per_staff_ex = capabilities_per_staff_ex.set_index(
'FacilityID_'
@@ -1055,7 +1055,7 @@ def _rescale_capabilities_to_capture_effective_capability(self):
)
if rescaling_factor > 1 and rescaling_factor != float("inf"):
self._daily_capabilities[officer] *= rescaling_factor
-
+
# We assume that increased daily capabilities is a result of each staff performing more
# daily patient facing time per day than contracted (or equivalently performing appts more
# efficiently).
diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py
index d6455cc861..a132c6e008 100644
--- a/src/tlo/methods/hiv.py
+++ b/src/tlo/methods/hiv.py
@@ -23,33 +23,29 @@
* Cotrimoxazole is not included - either in effect of consumption of the drug (because the effect is not known).
* Calibration has not been done: most things look OK - except HIV-AIDS deaths
"""
-from __future__ import annotations
import os
-from typing import TYPE_CHECKING, List
+from typing import List
import numpy as np
import pandas as pd
-from tlo import DAYS_IN_YEAR, Date, DateOffset, Module, Parameter, Property, Types, logging
+from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata, demography, tb
from tlo.methods.causes import Cause
from tlo.methods.dxmanager import DxTest
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
from tlo.util import create_age_range_lookup
-if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
-
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class Hiv(Module, GenericFirstAppointmentsMixin):
+class Hiv(Module):
"""
The HIV Disease Module
"""
@@ -397,19 +393,6 @@ def __init__(self, name=None, resourcefilepath=None, run_with_checks=False):
"length in days of inpatient stay for end-of-life HIV patients: list has two elements [low-bound-inclusive,"
" high-bound-exclusive]",
),
- # ------------------ scale-up parameters for scenario analysis ------------------ #
- "type_of_scaleup": Parameter(
- Types.STRING, "argument to determine type scale-up of program which will be implemented, "
- "can be 'none', 'target' or 'max'",
- ),
- "scaleup_start_year": Parameter(
- Types.INT,
- "the year when the scale-up starts (it will occur on 1st January of that year)"
- ),
- "scaleup_parameters": Parameter(
- Types.DATA_FRAME,
- "the parameters and values changed in scenario analysis"
- ),
}
def read_parameters(self, data_folder):
@@ -447,9 +430,6 @@ def read_parameters(self, data_folder):
# Load spectrum estimates of treatment cascade
p["treatment_cascade"] = workbook["spectrum_treatment_cascade"]
- # load parameters for scale-up projections
- p['scaleup_parameters'] = workbook["scaleup_parameters"]
-
# DALY weights
# get the DALY weight that this module will use from the weight database (these codes are just random!)
if "HealthBurden" in self.sim.modules.keys():
@@ -472,13 +452,10 @@ def read_parameters(self, data_folder):
)
def pre_initialise_population(self):
- """Do things required before the population is created
- * Build the LinearModels"""
- self._build_linear_models()
-
- def _build_linear_models(self):
- """Establish the Linear Models"""
-
+ """
+ * Establish the Linear Models
+ *
+ """
p = self.parameters
# ---- LINEAR MODELS -----
@@ -913,12 +890,6 @@ def initialise_simulation(self, sim):
# 2) Schedule the Logging Event
sim.schedule_event(HivLoggingEvent(self), sim.date + DateOffset(years=1))
- # Optional: Schedule the scale-up of programs
- if self.parameters["type_of_scaleup"] != 'none':
- scaleup_start_date = Date(self.parameters["scaleup_start_year"], 1, 1)
- assert scaleup_start_date >= self.sim.start_date, f"Date {scaleup_start_date} is before simulation starts."
- sim.schedule_event(HivScaleUpEvent(self), scaleup_start_date)
-
# 3) Determine who has AIDS and impose the Symptoms 'aids_symptoms'
# Those on ART currently (will not get any further events scheduled):
@@ -1042,37 +1013,31 @@ def initialise_simulation(self, sim):
self.item_codes_for_consumables_required['circ'] = \
hs.get_item_codes_from_package_name("Male circumcision ")
- # adult prep: 1 tablet daily
- self.item_codes_for_consumables_required['prep'] = \
- hs.get_item_code_from_item_name("Tenofovir (TDF)/Emtricitabine (FTC), tablet, 300/200 mg")
+ self.item_codes_for_consumables_required['prep'] = {
+ hs.get_item_code_from_item_name("Tenofovir (TDF)/Emtricitabine (FTC), tablet, 300/200 mg"): 1}
- # infant NVP 1.5mg daily for birth weight 2500g or above, for 6 weeks
- self.item_codes_for_consumables_required['infant_prep'] = \
- hs.get_item_code_from_item_name("Nevirapine, oral solution, 10 mg/ml")
+ # infant NVP given in 3-monthly dosages
+ self.item_codes_for_consumables_required['infant_prep'] = {
+ hs.get_item_code_from_item_name("Nevirapine, oral solution, 10 mg/ml"): 1}
# First - line ART for adults(age > "ART_age_cutoff_older_child")
- # TDF/3TC/DTG 120/60/50mg, 1 tablet per day
- # cotrim adult tablet, 1 tablet per day, units specified in mg * dispensation days
- self.item_codes_for_consumables_required['First-line ART regimen: adult'] = \
- hs.get_item_code_from_item_name("First-line ART regimen: adult")
- self.item_codes_for_consumables_required['First-line ART regimen: adult: cotrimoxazole'] = \
- hs.get_item_code_from_item_name("Cotrimoxizole, 960mg pppy")
+ self.item_codes_for_consumables_required['First-line ART regimen: adult'] = {
+ hs.get_item_code_from_item_name("First-line ART regimen: adult"): 1}
+ self.item_codes_for_consumables_required['First-line ART regimen: adult: cotrimoxazole'] = {
+ hs.get_item_code_from_item_name("Cotrimoxizole, 960mg pppy"): 1}
# ART for older children aged ("ART_age_cutoff_younger_child" < age <= "ART_age_cutoff_older_child"):
- # ABC/3TC/DTG 120/60/50mg, 3 tablets per day
- # cotrim paediatric tablet, 4 tablets per day, units specified in mg * dispensation days
- self.item_codes_for_consumables_required['First line ART regimen: older child'] = \
- hs.get_item_code_from_item_name("First line ART regimen: older child")
- self.item_codes_for_consumables_required['First line ART regimen: older child: cotrimoxazole'] = \
- hs.get_item_code_from_item_name("Cotrimoxazole 120mg_1000_CMST")
+ # cotrim is separate item - optional in get_cons call
+ self.item_codes_for_consumables_required['First line ART regimen: older child'] = {
+ hs.get_item_code_from_item_name("First line ART regimen: older child"): 1}
+ self.item_codes_for_consumables_required['First line ART regimen: older child: cotrimoxazole'] = {
+ hs.get_item_code_from_item_name("Sulfamethoxazole + trimethropin, tablet 400 mg + 80 mg"): 1}
# ART for younger children aged (age < "ART_age_cutoff_younger_child"):
- # ABC/3TC/DTG 120/60/10mg, 2 tablets per day
- # cotrim paediatric tablet, 2 tablets per day, units specified in mg * dispensation days
- self.item_codes_for_consumables_required['First line ART regimen: young child'] = \
- hs.get_item_code_from_item_name("First line ART regimen: young child")
- self.item_codes_for_consumables_required['First line ART regimen: young child: cotrimoxazole'] = \
- hs.get_item_code_from_item_name("Cotrimoxazole 120mg_1000_CMST")
+ self.item_codes_for_consumables_required['First line ART regimen: young child'] = {
+ hs.get_item_code_from_item_name("First line ART regimen: young child"): 1}
+ self.item_codes_for_consumables_required['First line ART regimen: young child: cotrimoxazole'] = {
+ hs.get_item_code_from_item_name("Sulfamethoxazole + trimethropin, oral suspension, 240 mg, 100 ml"): 1}
# 7) Define the DxTests
# HIV Rapid Diagnostic Test:
@@ -1101,49 +1066,6 @@ def initialise_simulation(self, sim):
)
)
- def update_parameters_for_program_scaleup(self):
- """ options for program scale-up are 'target' or 'max' """
- p = self.parameters
- scaled_params_workbook = p["scaleup_parameters"]
-
- if p['type_of_scaleup'] == 'target':
- scaled_params = scaled_params_workbook.set_index('parameter')['target_value'].to_dict()
- else:
- scaled_params = scaled_params_workbook.set_index('parameter')['max_value'].to_dict()
-
- # scale-up HIV program
- # reduce risk of HIV - applies to whole adult population
- p["beta"] = p["beta"] * scaled_params["reduction_in_hiv_beta"]
-
- # increase PrEP coverage for FSW after HIV test
- p["prob_prep_for_fsw_after_hiv_test"] = scaled_params["prob_prep_for_fsw_after_hiv_test"]
-
- # prep poll for AGYW - target to the highest risk
- # increase retention to 75% for FSW and AGYW
- p["prob_prep_for_agyw"] = scaled_params["prob_prep_for_agyw"]
- p["probability_of_being_retained_on_prep_every_3_months"] = scaled_params["probability_of_being_retained_on_prep_every_3_months"]
-
- # perfect retention on ART
- p["probability_of_being_retained_on_art_every_3_months"] = scaled_params["probability_of_being_retained_on_art_every_3_months"]
-
- # increase probability of VMMC after hiv test
- p["prob_circ_after_hiv_test"] = scaled_params["prob_circ_after_hiv_test"]
-
- # increase testing/diagnosis rates, default 2020 0.03/0.25 -> 93% dx
- p["hiv_testing_rates"]["annual_testing_rate_adults"] = scaled_params["annual_testing_rate_adults"]
-
- # ANC testing - value for mothers and infants testing
- p["prob_hiv_test_at_anc_or_delivery"] = scaled_params["prob_hiv_test_at_anc_or_delivery"]
- p["prob_hiv_test_for_newborn_infant"] = scaled_params["prob_hiv_test_for_newborn_infant"]
-
- # viral suppression rates
- # adults already at 95% by 2020
- # change all column values
- p["prob_start_art_or_vs"]["virally_suppressed_on_art"] = scaled_params["virally_suppressed_on_art"]
-
- # update exising linear models to use new scaled-up paramters
- self._build_linear_models()
-
def on_birth(self, mother_id, child_id):
"""
* Initialise our properties for a newborn individual;
@@ -1640,23 +1562,22 @@ def is_subset(col_for_set, col_for_subset):
def do_at_generic_first_appt(
self,
- person_id: int,
+ patient_id: int,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
# 'Automatic' testing for HIV for everyone attending care with AIDS symptoms:
# - suppress the footprint (as it done as part of another appointment)
# - do not do referrals if the person is HIV negative (assumed not time for counselling etc).
if "aids_symptoms" in symptoms:
event = HSI_Hiv_TestAndRefer(
- person_id=person_id,
+ person_id=patient_id,
module=self,
referred_from="hsi_generic_first_appt",
suppress_footprint=True,
do_not_refer_if_neg=True,
)
- schedule_hsi_event(event, priority=0, topen=self.sim.date)
+ self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date)
# ---------------------------------------------------------------------------
# Main Polling Event
@@ -2282,20 +2203,6 @@ def apply(self, person_id):
)
-class HivScaleUpEvent(Event, PopulationScopeEventMixin):
- """ This event exists to change parameters or functions
- depending on the scenario for projections which has been set
- It only occurs once on date: scaleup_start_date,
- called by initialise_simulation
- """
-
- def __init__(self, module):
- super().__init__(module)
-
- def apply(self, population):
- self.module.update_parameters_for_program_scaleup()
-
-
# ---------------------------------------------------------------------------
# Health System Interactions (HSI)
# ---------------------------------------------------------------------------
@@ -2501,10 +2408,6 @@ def apply(self, person_id, squeeze_factor):
# Update circumcision state
df.at[person_id, "li_is_circ"] = True
- # Add used equipment
- self.add_equipment({'Drip stand', 'Stool, adjustable height', 'Autoclave',
- 'Bipolar Diathermy Machine', 'Bed, adult', 'Trolley, patient'})
-
# Schedule follow-up appts
# schedule first follow-up appt, 3 days from procedure;
self.sim.modules["HealthSystem"].schedule_hsi_event(
@@ -2564,9 +2467,7 @@ def apply(self, person_id, squeeze_factor):
return self.sim.modules["HealthSystem"].get_blank_appt_footprint()
# Check that infant prophylaxis is available and if it is, initiate:
- if self.get_consumables(
- item_codes={self.module.item_codes_for_consumables_required['infant_prep']: 63}
- ):
+ if self.get_consumables(item_codes=self.module.item_codes_for_consumables_required['infant_prep']):
df.at[person_id, "hv_is_on_prep"] = True
# Schedule follow-up visit for 3 months time
@@ -2650,10 +2551,7 @@ def apply(self, person_id, squeeze_factor):
return self.make_appt_footprint({"Over5OPD": 1, "VCTPositive": 1})
# Check that PrEP is available and if it is, initiate or continue PrEP:
- quantity_required = self.module.parameters['dispensation_period_months'] * 30
- if self.get_consumables(
- item_codes={self.module.item_codes_for_consumables_required['prep']: quantity_required}
- ):
+ if self.get_consumables(item_codes=self.module.item_codes_for_consumables_required['prep']):
df.at[person_id, "hv_is_on_prep"] = True
# Schedule 'decision about whether to continue on PrEP' for 3 months time
@@ -2894,33 +2792,29 @@ def get_drugs(self, age_of_person):
whether individual drugs were available"""
p = self.module.parameters
- dispensation_days = 30 * self.module.parameters['dispensation_period_months']
if age_of_person < p["ART_age_cutoff_young_child"]:
# Formulation for young children
drugs_available = self.get_consumables(
- item_codes={self.module.item_codes_for_consumables_required[
- 'First line ART regimen: young child']: dispensation_days * 2},
- optional_item_codes={self.module.item_codes_for_consumables_required[
- 'First line ART regimen: young child: cotrimoxazole']: dispensation_days * 240},
+ item_codes=self.module.item_codes_for_consumables_required['First line ART regimen: young child'],
+ optional_item_codes=self.module.item_codes_for_consumables_required[
+ 'First line ART regimen: young child: cotrimoxazole'],
return_individual_results=True)
elif age_of_person <= p["ART_age_cutoff_older_child"]:
# Formulation for older children
drugs_available = self.get_consumables(
- item_codes={self.module.item_codes_for_consumables_required[
- 'First line ART regimen: older child']: dispensation_days * 3},
- optional_item_codes={self.module.item_codes_for_consumables_required[
- 'First line ART regimen: older child: cotrimoxazole']: dispensation_days * 480},
+ item_codes=self.module.item_codes_for_consumables_required['First line ART regimen: older child'],
+ optional_item_codes=self.module.item_codes_for_consumables_required[
+ 'First line ART regimen: older child: cotrimoxazole'],
return_individual_results=True)
else:
# Formulation for adults
drugs_available = self.get_consumables(
- item_codes={self.module.item_codes_for_consumables_required[
- 'First-line ART regimen: adult']: dispensation_days},
- optional_item_codes={self.module.item_codes_for_consumables_required[
- 'First-line ART regimen: adult: cotrimoxazole']: dispensation_days * 960},
+ item_codes=self.module.item_codes_for_consumables_required['First-line ART regimen: adult'],
+ optional_item_codes=self.module.item_codes_for_consumables_required[
+ 'First-line ART regimen: adult: cotrimoxazole'],
return_individual_results=True)
# add drug names to dict
@@ -3347,15 +3241,15 @@ def treatment_counts(subset):
count = sum(subset)
# proportion of subset living with HIV that are diagnosed:
proportion_diagnosed = (
- sum(subset & df.hv_diagnosed) / count if count > 0 else 0.0
+ sum(subset & df.hv_diagnosed) / count if count > 0 else 0
)
# proportions of subset living with HIV on treatment:
art = sum(subset & (df.hv_art != "not"))
- art_cov = art / count if count > 0 else 0.0
+ art_cov = art / count if count > 0 else 0
# proportion of subset on treatment that have good VL suppression
art_vs = sum(subset & (df.hv_art == "on_VL_suppressed"))
- art_cov_vs = art_vs / art if art > 0 else 0.0
+ art_cov_vs = art_vs / art if art > 0 else 0
return proportion_diagnosed, art_cov, art_cov_vs
alive_infected = df.is_alive & df.hv_inf
diff --git a/src/tlo/methods/hsi_generic_first_appts.py b/src/tlo/methods/hsi_generic_first_appts.py
index 37f6c5e261..0e912b8f06 100644
--- a/src/tlo/methods/hsi_generic_first_appts.py
+++ b/src/tlo/methods/hsi_generic_first_appts.py
@@ -1,248 +1,156 @@
-"""Events which describes the first interaction with the health system.
-
-This module contains the HSI events that represent the first contact with the health
-system, which are triggered by the onset of symptoms. Non-emergency symptoms lead to
-:py:class:`HSI_GenericNonEmergencyFirstAppt` and emergency symptoms lead to
-:py:class:`HSI_GenericEmergencyFirstAppt`.
"""
+The file contains the event HSI_GenericFirstApptAtFacilityLevel1, which describes the first interaction with
+the health system following the onset of acute generic symptoms.
+This file contains the HSI events that represent the first contact with the Health System, which are triggered by
+the onset of symptoms. Non-emergency symptoms lead to `HSI_GenericFirstApptAtFacilityLevel0` and emergency symptoms
+lead to `HSI_GenericEmergencyFirstApptAtFacilityLevel1`.
+"""
from __future__ import annotations
-from collections.abc import Callable
-from typing import TYPE_CHECKING, Any, Dict, List, Protocol, Set, Union
+from typing import TYPE_CHECKING, Literal, OrderedDict
-import numpy as np
-
-from tlo import Date, Module, logging
+from tlo import logging
from tlo.events import IndividualScopeEventMixin
from tlo.methods.hsi_event import HSI_Event
if TYPE_CHECKING:
- from typing import Optional, TypeAlias
-
+ from tlo import Module
from tlo.methods.dxmanager import DiagnosisTestReturnType
- from tlo.population import IndividualProperties
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-
-DiagnosisFunction: TypeAlias = Callable[[str, bool, bool], Any]
-ConsumablesChecker: TypeAlias = Callable[
- [
- Union[None, np.integer, int, List, Set, Dict],
- Union[None, np.integer, int, List, Set, Dict],
- ],
- Union[bool, Dict],
-]
-
-
-class HSIEventScheduler(Protocol):
-
- def __call__(
- self,
- hsi_event: HSI_Event,
- priority: int,
- topen: Date,
- tclose: Optional[Date] = None,
- ) -> None: ...
-
-
-class GenericFirstAppointmentsMixin:
- """Mix-in for modules with actions to perform on generic first appointments."""
-
- def do_at_generic_first_appt(
- self,
- *,
- person_id: int,
- individual_properties: IndividualProperties,
- symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
- diagnosis_function: DiagnosisFunction,
- consumables_checker: ConsumablesChecker,
- facility_level: str,
- treatment_id: str,
- ) -> None:
- """
- Actions to take during a non-emergency generic health system interaction (HSI).
-
- Derived classes should overwrite this method so that they are compatible with
- the :py:class:`~.HealthSystem` module, and can schedule HSI events when a
- individual presents symptoms indicative of the corresponding illness or
- condition.
-
- When overwriting, arguments that are not required can be left out of the
- definition. If done so, the method **must** take a ``**kwargs`` input to avoid
- errors when looping over all disease modules and running their generic HSI
- methods.
-
- HSI events should be scheduled by the :py:class:`Module` subclass implementing
- this method using the ``schedule_hsi_event`` argument.
-
- Implementations of this method should **not** make any updates to the population
- dataframe directly - if the target individuals properties need to be updated
- this should be performed by updating the ``individual_properties`` argument.
-
- :param person_id: Row index (ID) of the individual target of the HSI event in
- the population dataframe.
- :param individual_properties: Properties of individual target as provided in the
- population dataframe. Updates to individual properties may be written to
- this object.
- :param symptoms: List of symptoms the patient is experiencing.
- :param schedule_hsi_event: A function that can schedule subsequent HSI events.
- :param diagnosis_function: A function that can run diagnosis tests based on the
- patient's symptoms.
- :param consumables_checker: A function that can query the health system to check
- for available consumables.
- :param facility_level: The level of the facility that the patient presented at.
- :param treatment_id: The treatment id of the HSI event triggering the generic
- appointment.
- """
-
- def do_at_generic_first_appt_emergency(
- self,
- *,
- person_id: int,
- individual_properties: IndividualProperties,
- symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
- diagnosis_function: DiagnosisFunction,
- consumables_checker: ConsumablesChecker,
- facility_level: str,
- treatment_id: str,
- ) -> None:
- """
- Actions to take during an emergency generic health system interaction (HSI).
-
- Call signature is identical to the
- :py:meth:`~GenericFirstAppointmentsMixin.do_at_generic_first_appt` method.
-
- Derived classes should overwrite this method so that they are compatible with
- the :py:class`~.HealthSystem` module, and can schedule HSI events when a
- individual presents symptoms indicative of the corresponding illness or
- condition.
- """
-
-
-class _BaseHSIGenericFirstAppt(HSI_Event, IndividualScopeEventMixin):
+class HSI_BaseGenericFirstAppt(HSI_Event, IndividualScopeEventMixin):
+ """
+ """
+ MODULE_METHOD_ON_APPLY: Literal[
+ "do_at_generic_first_appt", "do_at_generic_first_appt_emergency"
+ ]
def __init__(self, module, person_id) -> None:
super().__init__(module, person_id=person_id)
- # No footprint, as this HSI (mostly just) determines which further HSI will be
- # needed for this person. In some cases, small bits of care are provided (e.g. a
- # diagnosis, or the provision of inhaler).
- self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({})
+ # No footprint, as this HSI (mostly just) determines which
+ # further HSI will be needed for this person. In some cases,
+ # small bits of care are provided (e.g. a diagnosis, or the
+ # provision of inhaler).
+ self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint(
+ {}
+ )
def _diagnosis_function(
self, tests, use_dict: bool = False, report_tried: bool = False
) -> DiagnosisTestReturnType:
"""
- Passed to modules when determining HSI events to be scheduled based on
- this generic appointment. Intended as the ``diagnosis_function`` argument to
- :py:meth:`GenericFirstAppointmentsMixin.do_at_generic_first_appt` or
- :py:meth:`GenericFirstAppointmentsMixin.do_at_generic_first_appt_emergency`.
+ Passed to modules when determining HSI_Events to be scheduled based on
+ this generic appointment. Intended as the diagnosis_function argument to the
+ Module.do_at_generic_{non_}_emergency.
Class-level definition avoids the need to redefine this method each time
- the :py:meth:`apply` method is called.
+ the .apply() method is called.
:param tests: The name of the test(s) to run via the diagnosis manager.
- :param use_dict_for_single: If ``True``, the return type will be a dictionary
- even if only one test was requested.
+ :param use_dict_for_single: If True, the return type will be a dictionary
+ even if only one test was requested.
:param report_dxtest_tried: Report if a test was attempted but could not
- be carried out due to for example lack of consumables, etc.
+ be carried out due to EG lack of consumables, etc.
:returns: Test results as dictionary key/value pairs.
"""
- return self.sim.modules["HealthSystem"].dx_manager.run_dx_test(
+ return self.healthcare_system.dx_manager.run_dx_test(
tests,
hsi_event=self,
use_dict_for_single=use_dict,
report_dxtest_tried=report_tried,
)
- @staticmethod
- def _do_at_generic_first_appt_for_module(module: Module) -> Callable:
- """Retrieves relevant do_at_generic_first_appt* method for a module.
-
- Must be implemented by concrete classes derived from this base class.
+ def _do_on_generic_first_appt(self, squeeze_factor: float = 0.) -> None:
+ """
"""
- raise NotImplementedError
+ # Make top-level reads of information, to avoid repeat accesses.
+ modules: OrderedDict[str, "Module"] = self.sim.modules
+ symptoms = modules["SymptomManager"].has_what(self.target)
- def apply(self, person_id: int, squeeze_factor: float = 0.0) -> None:
+ # Dynamically create immutable container with the target's details stored.
+ # This will avoid repeat DataFrame reads when we call the module-level functions.
+ df = self.sim.population.props
+ patient_details = self.sim.population.row_in_readonly_form(self.target)
+
+ proposed_patient_details_updates = {}
+
+ for module in modules.values():
+ module_patient_updates = getattr(module, self.MODULE_METHOD_ON_APPLY)(
+ patient_id=self.target,
+ patient_details=patient_details,
+ symptoms=symptoms,
+ diagnosis_function=self._diagnosis_function,
+ consumables_checker=self.get_consumables,
+ facility_level=self.ACCEPTED_FACILITY_LEVEL,
+ treatment_id=self.TREATMENT_ID,
+ random_state=self.module.rng,
+ )
+ # Record any requested DataFrame updates, but do not implement yet
+ # NOTE: |= syntax is only available in Python >=3.9
+ if module_patient_updates:
+ proposed_patient_details_updates = {
+ **proposed_patient_details_updates,
+ **module_patient_updates,
+ }
+
+ # Perform any DataFrame updates that were requested, all in one go.
+ if proposed_patient_details_updates:
+ df.loc[
+ self.target, proposed_patient_details_updates.keys()
+ ] = proposed_patient_details_updates.values()
+
+ def apply(self, person_id, squeeze_factor=0.) -> None:
"""
- Run the actions required during the health system interaction (HSI).
+ Run the actions required during the HSI.
TODO: person_id is not needed any more - but would have to go through the
whole codebase to manually identify instances of this class to change call
syntax, and leave other HSI_Event-derived classes alone.
"""
- # Create a memoized view of target individuals' properties as a context manager
- # that will automatically synchronize any updates back to the population
- # dataframe on exit
- with self.sim.population.individual_properties(
- self.target, read_only=False
- ) as individual_properties:
- if not individual_properties["is_alive"]:
- return
- # Pre-evaluate symptoms for individual to avoid repeat accesses
- # Use the individual_properties context here to save independent DF lookups
- symptoms = self.sim.modules["SymptomManager"].has_what(
- individual_details=individual_properties
- )
- schedule_hsi_event = self.sim.modules["HealthSystem"].schedule_hsi_event
- for module in self.sim.modules.values():
- if isinstance(module, GenericFirstAppointmentsMixin):
- self._do_at_generic_first_appt_for_module(module)(
- person_id=self.target,
- individual_properties=individual_properties,
- symptoms=symptoms,
- schedule_hsi_event=schedule_hsi_event,
- diagnosis_function=self._diagnosis_function,
- consumables_checker=self.get_consumables,
- facility_level=self.ACCEPTED_FACILITY_LEVEL,
- treatment_id=self.TREATMENT_ID,
- )
-
-
-class HSI_GenericNonEmergencyFirstAppt(_BaseHSIGenericFirstAppt):
- """
- This is a health system interaction event that represents the first interaction with
- the health system following the onset of non-emergency symptom(s).
+ if self.target_is_alive:
+ self._do_on_generic_first_appt(squeeze_factor=squeeze_factor)
- It is generated by the :py:class:`~HealthSeekingBehaviour` module.
- By default, it occurs at level '0' but it could occur also at other levels.
-
- It uses the non-emergency generic first appointment methods of the disease modules
- to determine any follow-up events that need to be scheduled.
+class HSI_GenericNonEmergencyFirstAppt(HSI_BaseGenericFirstAppt):
+ """
+ This is a Health System Interaction Event that represents the
+ first interaction with the health system following the onset
+ of non-emergency symptom(s).
+
+ It is generated by the HealthSeekingBehaviour module.
+
+ By default, it occurs at level '0' but it could occur also at
+ other levels.
+
+ It uses the non-emergency generic first appointment methods of
+ the disease modules to determine any follow-up events that need
+ to be scheduled.
"""
+ MODULE_METHOD_ON_APPLY = "do_at_generic_first_appt"
- def __init__(self, module, person_id, facility_level="0"):
- super().__init__(
- module,
- person_id=person_id,
- )
+ def __init__(self, module, person_id, facility_level='0'):
+ super().__init__(module, person_id=person_id, )
- assert module is self.sim.modules["HealthSeekingBehaviour"]
+ assert module is self.sim.modules['HealthSeekingBehaviour']
- self.TREATMENT_ID = "FirstAttendance_NonEmergency"
+ self.TREATMENT_ID = 'FirstAttendance_NonEmergency'
self.ACCEPTED_FACILITY_LEVEL = facility_level
- @staticmethod
- def _do_at_generic_first_appt_for_module(
- module: GenericFirstAppointmentsMixin,
- ) -> Callable:
- return module.do_at_generic_first_appt
-
-class HSI_GenericEmergencyFirstAppt(_BaseHSIGenericFirstAppt):
+class HSI_GenericEmergencyFirstAppt(HSI_BaseGenericFirstAppt):
"""
- This is a health system interaction event that represents the generic appointment
- which is the first interaction with the health system following the onset of
- emergency symptom(s).
+ This is a Health System Interaction Event that represents
+ the generic appointment which is the first interaction with
+ the health system following the onset of emergency symptom(s).
- It uses the emergency generic first appointment methods of the disease modules to
- determine any follow-up events that need to be scheduled.
+ It uses the emergency generic first appointment methods of
+ the disease modules to determine any follow-up events that need
+ to be scheduled.
"""
+ MODULE_METHOD_ON_APPLY = "do_at_generic_first_appt_emergency"
def __init__(self, module, person_id):
super().__init__(module, person_id=person_id)
@@ -257,15 +165,8 @@ def __init__(self, module, person_id):
self.TREATMENT_ID = "FirstAttendance_Emergency"
self.ACCEPTED_FACILITY_LEVEL = "1b"
- @staticmethod
- def _do_at_generic_first_appt_for_module(
- module: GenericFirstAppointmentsMixin,
- ) -> Callable:
- return module.do_at_generic_first_appt_emergency
-
-
class HSI_EmergencyCare_SpuriousSymptom(HSI_Event, IndividualScopeEventMixin):
- """HSI event providing accident & emergency care on spurious emergency symptoms."""
+ """This is an HSI event that provides Accident & Emergency Care for a person that has spurious emergency symptom."""
def __init__(self, module, person_id, accepted_facility_level="1a"):
super().__init__(module, person_id=person_id)
diff --git a/src/tlo/methods/labour.py b/src/tlo/methods/labour.py
index 35081b7d27..dcd8527466 100644
--- a/src/tlo/methods/labour.py
+++ b/src/tlo/methods/labour.py
@@ -8,20 +8,20 @@
import scipy.stats
from tlo import Date, DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType
-from tlo.logging.helpers import get_dataframe_row_as_dict_for_logging
from tlo.methods import Metadata, labour_lm, pregnancy_helper_functions
from tlo.methods.causes import Cause
from tlo.methods.dxmanager import DxTest
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.postnatal_supervisor import PostnatalWeekOneMaternalEvent
from tlo.util import BitsetHandler
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
- from tlo.population import IndividualProperties
+ from numpy.random import RandomState
+
+ from tlo.population import PatientDetails
# Standard logger
@@ -37,7 +37,7 @@
logger_pn.setLevel(logging.INFO)
-class Labour(Module, GenericFirstAppointmentsMixin):
+class Labour(Module):
"""This is module is responsible for the process of labour, birth and the immediate postnatal period (up until
48hrs post birth). This model has a number of core functions including; initiating the onset of labour for women on
their pre-determined due date (or prior to this for preterm labour/admission for delivery), applying the incidence
@@ -691,180 +691,174 @@ def get_and_store_labour_item_codes(self):
This function defines the required consumables for each intervention delivered during this module and stores
them in a module level dictionary called within HSIs
"""
- ic = self.sim.modules['HealthSystem'].get_item_code_from_item_name
+ get_item_code_from_pkg = self.sim.modules['HealthSystem'].get_item_codes_from_package_name
+
+ get_list_of_items = pregnancy_helper_functions.get_list_of_items
- # ---------------------------------- BLOOD TEST EQUIPMENT ---------------------------------------------------
- self.item_codes_lab_consumables['blood_test_equipment'] = \
- {ic('Blood collecting tube, 5 ml'): 1,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1
- }
# ---------------------------------- IV DRUG ADMIN EQUIPMENT -------------------------------------------------
self.item_codes_lab_consumables['iv_drug_equipment'] = \
- {ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1
- }
+ get_list_of_items(self, ['Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box'])
+
+ # ---------------------------------- BLOOD TEST EQUIPMENT ---------------------------------------------------
+ self.item_codes_lab_consumables['blood_test_equipment'] = \
+ get_list_of_items(self, ['Blood collecting tube, 5 ml',
+ 'Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box'])
# ------------------------------------------ FULL BLOOD COUNT -------------------------------------------------
- self.item_codes_lab_consumables['full_blood_count'] = {ic('Complete blood count'): 1}
+ self.item_codes_lab_consumables['full_blood_count'] = get_list_of_items(self, ['Complete blood count'])
# -------------------------------------------- DELIVERY ------------------------------------------------------
# assuming CDK has blade, soap, cord tie
self.item_codes_lab_consumables['delivery_core'] = \
- {ic('Clean delivery kit'): 1,
- ic('Chlorhexidine 1.5% solution_5_CMST'): 20,
- }
+ get_list_of_items(self, ['Clean delivery kit',
+ 'Chlorhexidine 1.5% solution_5_CMST'])
self.item_codes_lab_consumables['delivery_optional'] = \
- {ic('Gauze, absorbent 90cm x 40m_each_CMST'): 30,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1,
- ic('Paracetamol, tablet, 500 mg'): 8000
- }
+ get_list_of_items(self, ['Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box',
+ 'Gauze, absorbent 90cm x 40m_each_CMST',
+ 'Paracetamol, tablet, 500 mg'])
# -------------------------------------------- CAESAREAN DELIVERY ------------------------------------------
self.item_codes_lab_consumables['caesarean_delivery_core'] = \
- {ic('Halothane (fluothane)_250ml_CMST'): 100,
- ic('Ceftriaxone 1g, PFR_each_CMST'): 2,
- ic('Metronidazole 200mg_1000_CMST'): 1, # todo: replace
- }
+ get_list_of_items(self, ['Halothane (fluothane)_250ml_CMST',
+ 'Ceftriaxone 1g, PFR_each_CMST',
+ 'Metronidazole 200mg_1000_CMST'])
self.item_codes_lab_consumables['caesarean_delivery_optional'] = \
- {ic('Scalpel blade size 22 (individually wrapped)_100_CMST'): 1,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Sodium chloride, injectable solution, 0,9 %, 500 ml'): 2000,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1,
- ic('Foley catheter'): 1,
- ic('Bag, urine, collecting, 2000 ml'): 1,
- ic('Paracetamol, tablet, 500 mg'): 8000,
- ic('Declofenac injection_each_CMST'): 2,
- ic("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 2000,
- }
+ get_list_of_items(self, ['Scalpel blade size 22 (individually wrapped)_100_CMST',
+ 'Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Paracetamol, tablet, 500 mg',
+ 'Declofenac injection_each_CMST',
+ 'Pethidine, 50 mg/ml, 2 ml ampoule',
+ 'Foley catheter',
+ 'Bag, urine, collecting, 2000 ml',
+ "ringer's lactate (Hartmann's solution), 1000 ml_12_IDA",
+ 'Sodium chloride, injectable solution, 0,9 %, 500 ml',
+ "Giving set iv administration + needle 15 drops/ml_each_CMST",
+ "Chlorhexidine 1.5% solution_5_CMST"])
# -------------------------------------------- OBSTETRIC SURGERY ----------------------------------------------
self.item_codes_lab_consumables['obstetric_surgery_core'] = \
- {ic('Halothane (fluothane)_250ml_CMST'): 100,
- ic('Ceftriaxone 1g, PFR_each_CMST'): 2,
- ic('Metronidazole 200mg_1000_CMST'): 1, # todo: replace
- }
+ get_list_of_items(self, ['Halothane (fluothane)_250ml_CMST',
+ 'Ceftriaxone 1g, PFR_each_CMST',
+ 'Metronidazole 200mg_1000_CMST'])
self.item_codes_lab_consumables['obstetric_surgery_optional'] = \
- {ic('Scalpel blade size 22 (individually wrapped)_100_CMST'): 1,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Sodium chloride, injectable solution, 0,9 %, 500 ml'): 2000,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1,
- ic('Foley catheter'): 1,
- ic('Bag, urine, collecting, 2000 ml'): 1,
- ic('Paracetamol, tablet, 500 mg'): 8000,
- ic('Declofenac injection_each_CMST'): 2,
- ic("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 2000,
- }
+ get_list_of_items(self, ['Scalpel blade size 22 (individually wrapped)_100_CMST',
+ 'Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Paracetamol, tablet, 500 mg',
+ 'Declofenac injection_each_CMST',
+ 'Pethidine, 50 mg/ml, 2 ml ampoule',
+ 'Foley catheter',
+ 'Bag, urine, collecting, 2000 ml',
+ "ringer's lactate (Hartmann's solution), 1000 ml_12_IDA",
+ 'Sodium chloride, injectable solution, 0,9 %, 500 ml',
+ "Giving set iv administration + needle 15 drops/ml_each_CMST"])
# -------------------------------------------- ABX FOR PROM -------------------------------------------------
self.item_codes_lab_consumables['abx_for_prom'] = \
- {ic('Benzathine benzylpenicillin, powder for injection, 2.4 million IU'): 8}
+ get_list_of_items(self, ['Benzathine benzylpenicillin, powder for injection, 2.4 million IU'])
# -------------------------------------------- ANTENATAL STEROIDS ---------------------------------------------
-
self.item_codes_lab_consumables['antenatal_steroids'] = \
- {ic('Dexamethasone 5mg/ml, 5ml_each_CMST'): 12}
+ get_list_of_items(self, ['Dexamethasone 5mg/ml, 5ml_each_CMST'])
# ------------------------------------- INTRAVENOUS ANTIHYPERTENSIVES ---------------------------------------
self.item_codes_lab_consumables['iv_antihypertensives'] = \
- {ic('Hydralazine, powder for injection, 20 mg ampoule'): 1}
+ get_list_of_items(self, ['Hydralazine, powder for injection, 20 mg ampoule'])
# --------------------------------------- ORAL ANTIHYPERTENSIVES ---------------------------------------------
self.item_codes_lab_consumables['oral_antihypertensives'] = \
- {ic('Methyldopa 250mg_1000_CMST'): 1}
+ get_list_of_items(self, ['Methyldopa 250mg_1000_CMST'])
# ---------------------------------- SEVERE PRE-ECLAMPSIA/ECLAMPSIA -----------------------------------------
self.item_codes_lab_consumables['magnesium_sulfate'] = \
- {ic('Magnesium sulfate, injection, 500 mg/ml in 10-ml ampoule'): 2}
+ get_list_of_items(self, ['Magnesium sulfate, injection, 500 mg/ml in 10-ml ampoule'])
self.item_codes_lab_consumables['eclampsia_management_optional'] = \
- {ic('Sodium chloride, injectable solution, 0,9 %, 500 ml'): 2000,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1,
- ic('Oxygen, 1000 liters, primarily with oxygen cylinders'): 23_040,
- ic('Complete blood count'): 1,
- ic('Blood collecting tube, 5 ml'): 1,
- ic('Foley catheter'): 1,
- ic('Bag, urine, collecting, 2000 ml'): 1,
- }
+ get_list_of_items(self, ['Misoprostol, tablet, 200 mcg',
+ 'Oxytocin, injection, 10 IU in 1 ml ampoule',
+ 'Sodium chloride, injectable solution, 0,9 %, 500 ml',
+ 'Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box',
+ 'Oxygen, 1000 liters, primarily with oxygen cylinders',
+ 'Complete blood count',
+ 'Foley catheter',
+ 'Bag, urine, collecting, 2000 ml'])
+
# ------------------------------------- OBSTRUCTED LABOUR ---------------------------------------------------
self.item_codes_lab_consumables['obstructed_labour'] = \
- {ic('Lidocaine HCl (in dextrose 7.5%), ampoule 2 ml'): 1,
- ic('Benzathine benzylpenicillin, powder for injection, 2.4 million IU'): 8,
- ic('Gentamycin, injection, 40 mg/ml in 2 ml vial'): 6,
- ic('Sodium chloride, injectable solution, 0,9 %, 500 ml'): 2000,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1,
- ic('Complete blood count'): 1,
- ic('Blood collecting tube, 5 ml'): 1,
- ic('Foley catheter'): 1,
- ic('Bag, urine, collecting, 2000 ml'): 1,
- ic('Paracetamol, tablet, 500 mg'): 8000,
- ic('Pethidine, 50 mg/ml, 2 ml ampoule'): 6,
- ic('Gauze, absorbent 90cm x 40m_each_CMST'): 30,
- ic('Suture pack'): 1,
- }
+ get_list_of_items(self, ['Lidocaine HCl (in dextrose 7.5%), ampoule 2 ml',
+ 'Benzylpenicillin 3g (5MU), PFR_each_CMST',
+ 'Gentamycin, injection, 40 mg/ml in 2 ml vial',
+ 'Sodium chloride, injectable solution, 0,9 %, 500 ml',
+ 'Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box',
+ 'Complete blood count',
+ 'Foley catheter',
+ 'Bag, urine, collecting, 2000 ml',
+ 'Paracetamol, tablet, 500 mg',
+ 'Pethidine, 50 mg/ml, 2 ml ampoule',
+ 'Gauze, absorbent 90cm x 40m_each_CMST',
+ 'Suture pack'])
+
# ------------------------------------- OBSTETRIC VACUUM ---------------------------------------------------
- self.item_codes_lab_consumables['vacuum'] = {ic('Vacuum, obstetric'): 1}
+ self.item_codes_lab_consumables['vacuum'] = get_list_of_items(self, ['Vacuum, obstetric'])
# ------------------------------------- MATERNAL SEPSIS -----------------------------------------------------
self.item_codes_lab_consumables['maternal_sepsis_core'] = \
- {ic('Benzylpenicillin 3g (5MU), PFR_each_CMST'): 8,
- ic('Gentamycin, injection, 40 mg/ml in 2 ml vial'): 6,
- }
+ get_list_of_items(self, ['Benzylpenicillin 3g (5MU), PFR_each_CMST',
+ 'Gentamycin, injection, 40 mg/ml in 2 ml vial'])
+ # 'Metronidazole, injection, 500 mg in 100 ml vial'])
self.item_codes_lab_consumables['maternal_sepsis_optional'] = \
- {ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Oxygen, 1000 liters, primarily with oxygen cylinders'): 23_040,
- ic('Paracetamol, tablet, 500 mg'): 8000,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Foley catheter'): 1,
- ic('Bag, urine, collecting, 2000 ml'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1,
- ic('Complete blood count'): 1,
- }
+ get_list_of_items(self, ['Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Oxygen, 1000 liters, primarily with oxygen cylinders',
+ 'Paracetamol, tablet, 500 mg',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST',
+ 'Foley catheter',
+ 'Bag, urine, collecting, 2000 ml',
+ 'Disposables gloves, powder free, 100 pieces per box',
+ 'Complete blood count'])
+
# ------------------------------------- ACTIVE MANAGEMENT THIRD STAGE ---------------------------------------
- self.item_codes_lab_consumables['amtsl'] = {ic('Oxytocin, injection, 10 IU in 1 ml ampoule'): 1}
+ self.item_codes_lab_consumables['amtsl'] = \
+ get_list_of_items(self, ['Oxytocin, injection, 10 IU in 1 ml ampoule'])
# ------------------------------------- POSTPARTUM HAEMORRHAGE ---------------------------------------
self.item_codes_lab_consumables['pph_core'] = \
- {ic('Oxytocin, injection, 10 IU in 1 ml ampoule'): 5}
+ get_list_of_items(self, ['Oxytocin, injection, 10 IU in 1 ml ampoule'])
self.item_codes_lab_consumables['pph_optional'] = \
- {ic('Misoprostol, tablet, 200 mcg'): 600,
- ic('Pethidine, 50 mg/ml, 2 ml ampoule'): 6,
- ic('Oxygen, 1000 liters, primarily with oxygen cylinders'): 23_040,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Foley catheter'): 1,
- ic('Bag, urine, collecting, 2000 ml'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1,
- ic('Complete blood count'): 1,
- }
+ get_list_of_items(self, ['Misoprostol, tablet, 200 mcg',
+ 'Pethidine, 50 mg/ml, 2 ml ampoule',
+ 'Oxygen, 1000 liters, primarily with oxygen cylinders',
+ 'Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Bag, urine, collecting, 2000 ml',
+ 'Foley catheter',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box',
+ 'Complete blood count'])
# ------------------------------------- BLOOD TRANSFUSION ---------------------------------------
- self.item_codes_lab_consumables['blood_transfusion'] = {ic('Blood, one unit'): 2}
+ self.item_codes_lab_consumables['blood_transfusion'] = get_list_of_items(self, ['Blood, one unit'])
# ------------------------------------------ FULL BLOOD COUNT -------------------------------------------------
- self.item_codes_lab_consumables['hb_test'] = {ic('Haemoglobin test (HB)'): 1}
+ self.item_codes_lab_consumables['hb_test'] = get_list_of_items(self, ['Haemoglobin test (HB)'])
# ---------------------------------- IRON AND FOLIC ACID ------------------------------------------------------
- # Dose changes at run time
self.item_codes_lab_consumables['iron_folic_acid'] = \
- {ic('Ferrous Salt + Folic Acid, tablet, 200 + 0.25 mg'): 1}
+ get_item_code_from_pkg('Ferrous Salt + Folic Acid, tablet, 200 + 0.25 mg')
# -------------------------------------------- RESUSCITATION ------------------------------------------
- self.item_codes_lab_consumables['resuscitation'] =\
- {ic('Infant resuscitator, clear plastic + mask + bag_each_CMST'): 1}
+ self.item_codes_lab_consumables['resuscitation'] = \
+ get_list_of_items(self, ['Infant resuscitator, clear plastic + mask + bag_each_CMST'])
def initialise_simulation(self, sim):
# Update self.current_parameters
@@ -1057,7 +1051,7 @@ def further_on_birth_labour(self, mother_id):
# log delivery setting
logger.info(key='delivery_setting_and_mode', data={'mother': mother_id,
- 'facility_type': str(mni[mother_id]['delivery_setting']),
+ 'facility_type': mni[mother_id]['delivery_setting'],
'mode': mni[mother_id]['mode_of_delivery']})
# Store only live births to a mother parity
@@ -1674,9 +1668,7 @@ def prophylactic_labour_interventions(self, hsi_event):
# If she has not already receive antibiotics, we check for consumables
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_lab_consumables['abx_for_prom'],
- opt_cons=self.item_codes_lab_consumables['iv_drug_equipment'])
+ self, hsi_event, self.item_codes_lab_consumables, core='abx_for_prom', optional='iv_drug_equipment')
# Then query if these consumables are available during this HSI And provide if available.
# Antibiotics for from reduce risk of newborn sepsis within the first
@@ -1690,9 +1682,8 @@ def prophylactic_labour_interventions(self, hsi_event):
mni[person_id]['labour_state'] == 'late_preterm_labour':
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_lab_consumables['antenatal_steroids'],
- opt_cons=self.item_codes_lab_consumables['iv_drug_equipment'])
+ self, hsi_event, self.item_codes_lab_consumables, core='antenatal_steroids',
+ optional='iv_drug_equipment')
# If available they are given. Antenatal steroids reduce a preterm newborns chance of developing
# respiratory distress syndrome and of death associated with prematurity
@@ -1754,9 +1745,8 @@ def assessment_and_treatment_of_severe_pre_eclampsia_mgso4(self, hsi_event, labo
# Define and check for the required consumables
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_lab_consumables['magnesium_sulfate'],
- opt_cons=self.item_codes_lab_consumables['eclampsia_management_optional'])
+ self, hsi_event, self.item_codes_lab_consumables, core='magnesium_sulfate',
+ optional='eclampsia_management_optional')
# If the consumables are available - the intervention is delivered. IV magnesium reduces the
# probability that a woman with severe pre-eclampsia will experience eclampsia in labour
@@ -1784,9 +1774,8 @@ def assessment_and_treatment_of_hypertension(self, hsi_event, labour_stage):
# Then query if these consumables are available during this HSI
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_lab_consumables['iv_antihypertensives'],
- opt_cons=self.item_codes_lab_consumables['iv_drug_equipment'])
+ self, hsi_event, self.item_codes_lab_consumables, core='iv_antihypertensives',
+ optional='iv_drug_equipment')
# If they are available then the woman is started on treatment. Intravenous antihypertensive reduce a
# womans risk of progression from mild to severe gestational hypertension ANd reduce risk of death for
@@ -1800,9 +1789,8 @@ def assessment_and_treatment_of_hypertension(self, hsi_event, labour_stage):
elif (labour_stage == 'pp') and (df.at[person_id, 'pn_htn_disorders'] == 'severe_gest_htn'):
df.at[person_id, 'pn_htn_disorders'] = 'gest_htn'
- dose = (7 * 4) * 6 # approximating 4 tablets a day, for 6 weeks
- cons = {_i: dose for _i in self.item_codes_lab_consumables['oral_antihypertensives']}
- avail = hsi_event.get_consumables(item_codes=cons)
+ avail = hsi_event.get_consumables(
+ item_codes=self.item_codes_lab_consumables['oral_antihypertensives'])
if avail:
df.at[person_id, 'la_gest_htn_on_treatment'] = True
@@ -1833,9 +1821,8 @@ def assessment_and_treatment_of_eclampsia(self, hsi_event, labour_stage):
# define and check required consumables
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_lab_consumables['magnesium_sulfate'],
- opt_cons=self.item_codes_lab_consumables['eclampsia_management_optional'])
+ self, hsi_event, self.item_codes_lab_consumables, core='magnesium_sulfate',
+ optional='eclampsia_management_optional')
if (labour_stage == 'ip') and (df.at[person_id, 'ac_admitted_for_immediate_delivery'] == 'none'):
self.determine_delivery_mode_in_spe_or_ec(person_id, hsi_event, 'ec')
@@ -1881,18 +1868,14 @@ def refer_for_cs():
# If the general package is available AND the facility has the correct tools to carry out the
# delivery then it can occur
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_lab_consumables['vacuum'],
- opt_cons=self.item_codes_lab_consumables['obstructed_labour'])
+ self, hsi_event, self.item_codes_lab_consumables, core='vacuum',
+ optional='obstructed_labour')
# run HCW check
sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='avd',
hsi_event=hsi_event)
if avail and sf_check:
- # Add used equipment
- hsi_event.add_equipment({'Delivery Forceps', 'Vacuum extractor'})
-
pregnancy_helper_functions.log_met_need(self, f'avd_{indication}', hsi_event)
# If AVD was successful then we record the mode of delivery. We use this variable to reduce
@@ -1938,9 +1921,8 @@ def assessment_and_treatment_of_maternal_sepsis(self, hsi_event, labour_stage):
# Define and check available consumables
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_lab_consumables['maternal_sepsis_core'],
- opt_cons=self.item_codes_lab_consumables['maternal_sepsis_optional'])
+ self, hsi_event, self.item_codes_lab_consumables, core='maternal_sepsis_core',
+ optional='maternal_sepsis_optional')
# If delivered this intervention reduces a womans risk of dying from sepsis
if avail and sf_check:
@@ -2018,9 +2000,7 @@ def active_management_of_the_third_stage_of_labour(self, hsi_event):
# Define and check available consumables
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_lab_consumables['amtsl'],
- opt_cons=self.item_codes_lab_consumables['iv_drug_equipment'])
+ self, hsi_event, self.item_codes_lab_consumables, core='amtsl', optional='iv_drug_equipment')
# run HCW check
sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='uterotonic',
@@ -2051,9 +2031,7 @@ def assessment_and_treatment_of_pph_uterine_atony(self, hsi_event):
# Define and check available consumables
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_lab_consumables['pph_core'],
- opt_cons=self.item_codes_lab_consumables['pph_optional'])
+ self, hsi_event, self.item_codes_lab_consumables, core='pph_core', optional='pph_optional')
# run HCW check
sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='uterotonic',
@@ -2136,28 +2114,23 @@ def surgical_management_of_pph(self, hsi_event):
# We log the required consumables and condition the surgery happening on the availability of the
# first consumable in this package, the anaesthetic required for the surgery
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_lab_consumables['obstetric_surgery_core'],
- opt_cons=self.item_codes_lab_consumables['obstetric_surgery_optional'])
+ self, hsi_event, self.item_codes_lab_consumables, core='obstetric_surgery_core',
+ optional='obstetric_surgery_optional')
# run HCW check
sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='surg',
hsi_event=hsi_event)
- if avail and sf_check:
- # Add used equipment
- hsi_event.add_equipment(hsi_event.healthcare_system.equipment.from_pkg_names('Major Surgery'))
+ # determine if uterine preserving surgery will be successful
+ treatment_success_pph = params['success_rate_pph_surgery'] > self.rng.random_sample()
- # determine if uterine preserving surgery will be successful
- treatment_success_pph = params['success_rate_pph_surgery'] > self.rng.random_sample()
+ # If resources are available and the surgery is a success then a hysterectomy does not occur
+ if treatment_success_pph and avail and sf_check:
+ self.pph_treatment.set(person_id, 'surgery')
- if treatment_success_pph:
- self.pph_treatment.set(person_id, 'surgery')
- else:
- # If the treatment is unsuccessful then women will require a hysterectomy to stop the bleeding
- hsi_event.add_equipment({'Hysterectomy set'})
- self.pph_treatment.set(person_id, 'hysterectomy')
- df.at[person_id, 'la_has_had_hysterectomy'] = True
+ elif not treatment_success_pph and avail and sf_check:
+ self.pph_treatment.set(person_id, 'hysterectomy')
+ df.at[person_id, 'la_has_had_hysterectomy'] = True
# log intervention delivery
if self.pph_treatment.has_all(person_id, 'surgery') or df.at[person_id, 'la_has_had_hysterectomy']:
@@ -2177,17 +2150,14 @@ def blood_transfusion(self, hsi_event):
# Check consumables
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_lab_consumables['blood_transfusion'],
- opt_cons=self.item_codes_lab_consumables['iv_drug_equipment'])
+ self, hsi_event, self.item_codes_lab_consumables, core='blood_transfusion', number=2,
+ optional='iv_drug_equipment')
# check HCW
sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self, sf='blood_tran',
hsi_event=hsi_event)
if avail and sf_check:
- hsi_event.add_equipment({'Drip stand', 'Infusion pump'})
-
mni[person_id]['received_blood_transfusion'] = True
pregnancy_helper_functions.log_met_need(self, 'blood_tran', hsi_event)
@@ -2211,9 +2181,6 @@ def assessment_and_treatment_of_anaemia(self, hsi_event):
mother = df.loc[person_id]
mni = self.sim.modules['PregnancySupervisor'].mother_and_newborn_info
- # Add used equipment
- hsi_event.add_equipment({'Analyser, Haematology'})
-
# Use dx_test function to assess anaemia status
test_result = self.sim.modules['HealthSystem'].dx_manager.run_dx_test(
dx_tests_to_run='full_blood_count_hb_pn', hsi_event=hsi_event)
@@ -2335,27 +2302,28 @@ def run_if_receives_comprehensive_emergency_obstetric_care_cant_run(self, hsi_ev
def do_at_generic_first_appt_emergency(
self,
- person_id: int,
- individual_properties: IndividualProperties,
- schedule_hsi_event: HSIEventScheduler,
+ patient_id: int,
+ patient_details: PatientDetails,
+ random_state: RandomState,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
mni = self.sim.modules["PregnancySupervisor"].mother_and_newborn_info
labour_list = self.sim.modules["Labour"].women_in_labour
- if person_id in labour_list:
- la_currently_in_labour = individual_properties["la_currently_in_labour"]
+ if patient_id in labour_list:
+ la_currently_in_labour = patient_details.la_currently_in_labour
if (
la_currently_in_labour
- & mni[person_id]["sought_care_for_complication"]
- & (mni[person_id]["sought_care_labour_phase"] == "intrapartum")
+ & mni[patient_id]["sought_care_for_complication"]
+ & (mni[patient_id]["sought_care_labour_phase"] == "intrapartum")
):
event = HSI_Labour_ReceivesSkilledBirthAttendanceDuringLabour(
module=self,
- person_id=person_id,
+ person_id=patient_id,
+ # facility_level_of_this_hsi=random_state.choice(["1a", "1b"]),
facility_level_of_this_hsi=self.rng.choice(["1a", "1b"]),
)
- schedule_hsi_event(
+ self.healthsystem.schedule_hsi_event(
event,
priority=0,
topen=self.sim.date,
@@ -2612,7 +2580,7 @@ def apply(self, individual_id):
self.module.set_intrapartum_complications(individual_id, complication=complication)
if df.at[individual_id, 'la_obstructed_labour']:
- logger.info(key='maternal_complication', data={'person': individual_id,
+ logger.info(key='maternal_complication', data={'mother': individual_id,
'type': 'obstructed_labour',
'timing': 'intrapartum'})
@@ -2936,14 +2904,8 @@ def apply(self, person_id, squeeze_factor):
# LOG CONSUMABLES FOR DELIVERY...
# We assume all deliveries require this basic package of consumables
avail = pregnancy_helper_functions.return_cons_avail(
- self.module, self,
- cons=self.module.item_codes_lab_consumables['delivery_core'],
- opt_cons=self.module.item_codes_lab_consumables['delivery_optional'])
-
- # Add used equipment
- self.add_equipment({'Delivery set', 'Weighing scale', 'Stethoscope, foetal, monaural, Pinard, plastic',
- 'Resuscitaire', 'Sphygmomanometer', 'Tray, emergency', 'Suction machine',
- 'Thermometer', 'Drip stand', 'Infusion pump'})
+ self.module, self, self.module.item_codes_lab_consumables, core='delivery_core',
+ optional='delivery_optional')
# If the clean delivery kit consumable is available, we assume women benefit from clean delivery
if avail:
@@ -2977,7 +2939,7 @@ def apply(self, person_id, squeeze_factor):
self.module.progression_of_hypertensive_disorders(person_id, property_prefix='ps')
if df.at[person_id, 'la_obstructed_labour']:
- logger.info(key='maternal_complication', data={'person': person_id,
+ logger.info(key='maternal_complication', data={'mother': person_id,
'type': 'obstructed_labour',
'timing': 'intrapartum'})
@@ -3019,7 +2981,7 @@ def apply(self, person_id, squeeze_factor):
# TODO: potential issue is that this consumable is being logged now for every birth as opposed to
# for each birth where resuscitation of the newborn is required
avail = pregnancy_helper_functions.return_cons_avail(
- self.module, self, cons=self.module.item_codes_lab_consumables['resuscitation'], opt_cons=None)
+ self.module, self, self.module.item_codes_lab_consumables, core='resuscitation')
# Run HCW check
sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.module,
@@ -3118,7 +3080,7 @@ def apply(self, person_id, squeeze_factor):
# log the PNC visit
logger.info(key='postnatal_check', data={'person_id': person_id,
- 'delivery_setting': str(mni[person_id]['delivery_setting']),
+ 'delivery_setting': mni[person_id]['delivery_setting'],
'visit_number': df.at[person_id, 'la_pn_checks_maternal'],
'timing': mni[person_id]['will_receive_pnc']})
@@ -3238,9 +3200,8 @@ def apply(self, person_id, squeeze_factor):
# We log the required consumables and condition the caesarean happening on the availability of the
# first consumable in this package, the anaesthetic required for the surgery
avail = pregnancy_helper_functions.return_cons_avail(
- self.module, self,
- cons=self.module.item_codes_lab_consumables['caesarean_delivery_core'],
- opt_cons=self.module.item_codes_lab_consumables['caesarean_delivery_optional'])
+ self.module, self, self.module.item_codes_lab_consumables, core='caesarean_delivery_core',
+ optional='caesarean_delivery_optional')
# We check that the HCW will deliver the intervention
sf_check = pregnancy_helper_functions.check_emonc_signal_function_will_run(self.module, sf='surg',
@@ -3251,13 +3212,8 @@ def apply(self, person_id, squeeze_factor):
logger.debug(key='message', data="cs delivery blocked for this analysis")
elif (avail and sf_check) or (mni[person_id]['cs_indication'] == 'other'):
- # If intervention is delivered - add used equipment
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery'))
-
- logger.info(
- key='caesarean_delivery',
- data=get_dataframe_row_as_dict_for_logging(df, person_id),
- )
+ person = df.loc[person_id]
+ logger.info(key='caesarean_delivery', data=person.to_dict())
logger.info(key='cs_indications', data={'id': person_id,
'indication': mni[person_id]['cs_indication']})
@@ -3289,7 +3245,6 @@ def apply(self, person_id, squeeze_factor):
# Unsuccessful repair will lead to this woman requiring a hysterectomy. Hysterectomy will also reduce
# risk of death from uterine rupture but leads to permanent infertility in the simulation
else:
- self.add_equipment({'Hysterectomy set'})
df.at[person_id, 'la_has_had_hysterectomy'] = True
# ============================= SURGICAL MANAGEMENT OF POSTPARTUM HAEMORRHAGE==================================
diff --git a/src/tlo/methods/malaria.py b/src/tlo/methods/malaria.py
index b1fdfb09dd..7731ea923c 100644
--- a/src/tlo/methods/malaria.py
+++ b/src/tlo/methods/malaria.py
@@ -11,26 +11,25 @@
import pandas as pd
-from tlo import Date, DateOffset, Module, Parameter, Property, Types, logging
+from tlo import DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import DiagnosisFunction, IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, Predictor
from tlo.methods import Metadata
from tlo.methods.causes import Cause
from tlo.methods.dxmanager import DxTest
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
from tlo.util import random_date
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import DiagnosisFunction, HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class Malaria(Module, GenericFirstAppointmentsMixin):
+class Malaria(Module):
def __init__(self, name=None, resourcefilepath=None):
"""Create instance of Malaria module
@@ -188,19 +187,8 @@ def __init__(self, name=None, resourcefilepath=None):
'prob_of_treatment_success': Parameter(
Types.REAL,
'probability that treatment will clear malaria symptoms'
- ),
- "type_of_scaleup": Parameter(
- Types.STRING, "argument to determine type scale-up of program which will be implemented, "
- "can be 'none', 'target' or 'max'",
- ),
- "scaleup_start_year": Parameter(
- Types.INT,
- "the year when the scale-up starts (it will occur on 1st January of that year)"
- ),
- "scaleup_parameters": Parameter(
- Types.DATA_FRAME,
- "the parameters and values changed in scenario analysis"
)
+
}
PROPERTIES = {
@@ -253,15 +241,11 @@ def read_parameters(self, data_folder):
p['sev_symp_prob'] = workbook['severe_symptoms']
p['rdt_testing_rates'] = workbook['WHO_TestData2023']
- p['highrisk_districts'] = workbook['highrisk_districts']
p['inf_inc'] = pd.read_csv(self.resourcefilepath / 'malaria' / 'ResourceFile_malaria_InfInc_expanded.csv')
p['clin_inc'] = pd.read_csv(self.resourcefilepath / 'malaria' / 'ResourceFile_malaria_ClinInc_expanded.csv')
p['sev_inc'] = pd.read_csv(self.resourcefilepath / 'malaria' / 'ResourceFile_malaria_SevInc_expanded.csv')
- # load parameters for scale-up projections
- p['scaleup_parameters'] = workbook["scaleup_parameters"]
-
# check itn projected values are <=0.7 and rounded to 1dp for matching to incidence tables
p['itn'] = round(p['itn'], 1)
assert (p['itn'] <= 0.7)
@@ -326,16 +310,13 @@ def read_parameters(self, data_folder):
)
def pre_initialise_population(self):
- """Do things required before the population is created
- * Build the LinearModels"""
- self._build_linear_models()
-
- def _build_linear_models(self):
- """Establish the Linear Models
+ """
+ * Establish the Linear Models
if HIV is registered, the conditional predictors will apply
otherwise only IPTp will affect risk of clinical/severe malaria
"""
+
p = self.parameters
# ---- LINEAR MODELS -----
@@ -374,7 +355,7 @@ def _build_linear_models(self):
p['rr_severe_malaria_hiv_over5']),
Predictor().when('(hv_inf == True) & (is_pregnant == True)',
p['rr_severe_malaria_hiv_pregnant']),
- ] if "Hiv" in self.sim.modules else []
+ ] if "hiv" in self.sim.modules else []
self.lm["rr_of_severe_malaria"] = LinearModel.multiplicative(
*(predictors + conditional_predictors))
@@ -552,12 +533,8 @@ def general_population_rdt_scheduler(self, population):
# extract annual testing rates from NMCP reports
# this is the # rdts issued divided by population size
- year = self.sim.date.year if self.sim.date.year <= 2024 else 2024
-
- test_rates = (
- p['rdt_testing_rates'].set_index('Year')['Rate_rdt_testing'].dropna()
- )
- rdt_rate = test_rates.loc[min(test_rates.index.max(), year)] / 12
+ test_rates = p['rdt_testing_rates'].set_index('Year')['Rate_rdt_testing'].dropna()
+ rdt_rate = test_rates.loc[min(test_rates.index.max(), self.sim.date.year)] / 12
# adjust rdt usage reported rate to reflect consumables availability
rdt_rate = rdt_rate * p['scaling_factor_for_rdt_availability']
@@ -600,12 +577,6 @@ def initialise_simulation(self, sim):
sim.schedule_event(MalariaTxLoggingEvent(self), sim.date + DateOffset(years=1))
sim.schedule_event(MalariaPrevDistrictLoggingEvent(self), sim.date + DateOffset(months=1))
- # Optional: Schedule the scale-up of programs
- if self.parameters["type_of_scaleup"] != 'none':
- scaleup_start_date = Date(self.parameters["scaleup_start_year"], 1, 1)
- assert scaleup_start_date >= self.sim.start_date, f"Date {scaleup_start_date} is before simulation starts."
- sim.schedule_event(MalariaScaleUpEvent(self), scaleup_start_date)
-
# 2) ----------------------------------- DIAGNOSTIC TESTS -----------------------------------
# Create the diagnostic test representing the use of RDT for malaria diagnosis
# and registers it with the Diagnostic Test Manager
@@ -654,62 +625,7 @@ def initialise_simulation(self, sim):
# malaria IPTp for pregnant women
self.item_codes_for_consumables_required['malaria_iptp'] = get_item_code(
- 'Sulfamethoxazole + trimethropin, tablet 400 mg + 80 mg'
- )
-
- def update_parameters_for_program_scaleup(self):
- """ options for program scale-up are 'target' or 'max' """
- p = self.parameters
- scaled_params_workbook = p["scaleup_parameters"]
-
- if p['type_of_scaleup'] == 'target':
- scaled_params = scaled_params_workbook.set_index('parameter')['target_value'].to_dict()
- else:
- scaled_params = scaled_params_workbook.set_index('parameter')['max_value'].to_dict()
-
- # scale-up malaria program
- # increase testing
- # prob_malaria_case_tests=0.4 default
- p["prob_malaria_case_tests"] = scaled_params["prob_malaria_case_tests"]
-
- # gen pop testing rates
- # annual Rate_rdt_testing=0.64 at 2023
- p["rdt_testing_rates"]["Rate_rdt_testing"] = scaled_params["rdt_testing_rates"]
-
- # treatment reaches XX
- # no default between testing and treatment, governed by tx availability
-
- # coverage IPTp reaches XX
- # given during ANC visits and MalariaIPTp Event which selects ALL eligible women
-
- # treatment success reaches 1 - default is currently 1 also
- p["prob_of_treatment_success"] = scaled_params["prob_of_treatment_success"]
-
- # bednet and ITN coverage
- # set IRS for 4 high-risk districts
- # lookup table created in malaria read_parameters
- # produces self.itn_irs called by malaria poll to draw incidence
- # need to overwrite this
- highrisk_distr_num = p["highrisk_districts"]["district_num"]
-
- # Find indices where District_Num is in highrisk_distr_num
- mask = self.itn_irs['irs_rate'].index.get_level_values('District_Num').isin(
- highrisk_distr_num)
-
- # IRS values can be 0 or 0.8 - no other value in lookup table
- self.itn_irs['irs_rate'].loc[mask] = scaled_params["irs_district"]
-
- # set ITN for all districts
- # Set these values to 0.7 - this is the max value possible in lookup table
- # equivalent to 0.7 of all pop sleeping under bednet
- # household coverage could be 100%, but not everyone in household sleeping under bednet
- self.itn_irs['itn_rate'] = scaled_params["itn_district"]
-
- # itn rates for 2019 onwards
- p["itn"] = scaled_params["itn"]
-
- # update exising linear models to use new scaled-up parameters
- self._build_linear_models()
+ 'Sulfamethoxazole + trimethropin, tablet 400 mg + 80 mg')
def on_birth(self, mother_id, child_id):
df = self.sim.population.props
@@ -759,7 +675,7 @@ def check_if_fever_is_caused_by_malaria(
self,
true_malaria_infection_type: str,
diagnosis_function: DiagnosisFunction,
- person_id: Optional[int] = None,
+ patient_id: Optional[int] = None,
fever_is_a_symptom: Optional[bool] = True,
patient_age: Optional[Union[int, float]] = None,
facility_level: Optional[str] = None,
@@ -778,14 +694,14 @@ def check_if_fever_is_caused_by_malaria(
# Log the test: line-list of summary information about each test
logger.info(
key="rdt_log",
- data=_data_for_rdt_log(
- person_id=person_id,
- age=patient_age,
- fever_is_a_symptom=fever_is_a_symptom,
- dx_result=dx_result,
- facility_level=facility_level,
- treatment_id=treatment_id
- )
+ data={
+ "person_id": patient_id,
+ "age": patient_age,
+ "fever_present": fever_is_a_symptom,
+ "rdt_result": dx_result,
+ "facility_level": facility_level,
+ "called_by": treatment_id,
+ },
)
# Severe malaria infection always returns positive RDT
@@ -798,15 +714,16 @@ def check_if_fever_is_caused_by_malaria(
def do_at_generic_first_appt(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
diagnosis_function: DiagnosisFunction,
facility_level: str,
treatment_id: str,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
+ patient_details_updates = {}
+
malaria_associated_symptoms = {
"fever",
"headache",
@@ -816,72 +733,75 @@ def do_at_generic_first_appt(
}
if (
bool(set(symptoms) & malaria_associated_symptoms)
- and individual_properties["ma_tx"] == "none"
+ and patient_details.ma_tx == "none"
):
malaria_test_result = self.check_if_fever_is_caused_by_malaria(
- true_malaria_infection_type=individual_properties["ma_inf_type"],
+ true_malaria_infection_type=patient_details.ma_inf_type,
diagnosis_function=diagnosis_function,
- person_id=person_id,
+ patient_id=patient_id,
fever_is_a_symptom="fever" in symptoms,
- patient_age=individual_properties["age_years"],
+ patient_age=patient_details.age_years,
facility_level=facility_level,
treatment_id=treatment_id,
)
# Treat / refer based on diagnosis
if malaria_test_result == "severe_malaria":
- individual_properties["ma_dx_counter"] += 1
- event = HSI_Malaria_Treatment_Complicated(person_id=person_id, module=self)
- schedule_hsi_event(
+ patient_details_updates["ma_dx_counter"] = patient_details.ma_dx_counter + 1
+ event = HSI_Malaria_Treatment_Complicated(person_id=patient_id, module=self)
+ self.healthsystem.schedule_hsi_event(
event, priority=0, topen=self.sim.date
)
# return type 'clinical_malaria' includes asymptomatic infection
elif malaria_test_result == "clinical_malaria":
- individual_properties["ma_dx_counter"] += 1
- event = HSI_Malaria_Treatment(person_id=person_id, module=self)
- schedule_hsi_event(
+ patient_details_updates["ma_dx_counter"] = patient_details.ma_dx_counter + 1
+ event = HSI_Malaria_Treatment(person_id=patient_id, module=self)
+ self.healthsystem.schedule_hsi_event(
event, priority=1, topen=self.sim.date
)
+ return patient_details_updates
def do_at_generic_first_appt_emergency(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
diagnosis_function: DiagnosisFunction,
facility_level: str,
treatment_id: str,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
# This is called for a person (of any age) that attends an
# emergency generic HSI and has a fever.
# (Quick diagnosis algorithm - just perfectly recognises the
# symptoms of severe malaria.)
+ patient_details_updates = {}
+
if 'severe_malaria' in symptoms:
- if individual_properties["ma_tx"] == 'none':
+ if patient_details.ma_tx == 'none':
# Check if malaria parasitaemia:
malaria_test_result = self.check_if_fever_is_caused_by_malaria(
- true_malaria_infection_type=individual_properties["ma_inf_type"],
+ true_malaria_infection_type=patient_details.ma_inf_type,
diagnosis_function=diagnosis_function,
- person_id=person_id,
+ patient_id=patient_id,
fever_is_a_symptom="fever" in symptoms,
- patient_age=individual_properties["age_years"],
+ patient_age=patient_details.age_years,
facility_level=facility_level,
treatment_id=treatment_id,
)
# if any symptoms indicative of malaria and they have parasitaemia (would return a positive rdt)
if malaria_test_result in ('severe_malaria', 'clinical_malaria'):
- individual_properties['ma_dx_counter'] += 1
+ patient_details_updates['ma_dx_counter'] = patient_details.ma_dx_counter + 1
# Launch the HSI for treatment for Malaria, HSI_Malaria_Treatment will determine correct treatment
event = HSI_Malaria_Treatment_Complicated(
- person_id=person_id, module=self,
+ person_id=patient_id, module=self,
)
- schedule_hsi_event(
+ self.healthsystem.schedule_hsi_event(
event, priority=0, topen=self.sim.date
)
+ return patient_details_updates
class MalariaPollingEventDistrict(RegularEvent, PopulationScopeEventMixin):
"""
@@ -902,21 +822,6 @@ def apply(self, population):
self.module.general_population_rdt_scheduler(population)
-class MalariaScaleUpEvent(Event, PopulationScopeEventMixin):
- """ This event exists to change parameters or functions
- depending on the scenario for projections which has been set
- It only occurs once on date: scaleup_start_date,
- called by initialise_simulation
- """
-
- def __init__(self, module):
- super().__init__(module)
-
- def apply(self, population):
-
- self.module.update_parameters_for_program_scaleup()
-
-
class MalariaIPTp(RegularEvent, PopulationScopeEventMixin):
"""
malaria prophylaxis for pregnant women
@@ -1068,15 +973,15 @@ def apply(self, person_id, squeeze_factor):
)
# Log the test: line-list of summary information about each test
- fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id=person_id)
- person_details_for_test = _data_for_rdt_log(
- person_id=person_id,
- age=df.at[person_id, 'age_years'],
- fever_is_a_symptom=fever_present,
- dx_result=dx_result,
- facility_level=self.ACCEPTED_FACILITY_LEVEL,
- treatment_id=self.TREATMENT_ID,
- )
+ fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id)
+ person_details_for_test = {
+ 'person_id': person_id,
+ 'age': df.at[person_id, 'age_years'],
+ 'fever_present': fever_present,
+ 'rdt_result': dx_result,
+ 'facility_level': self.ACCEPTED_FACILITY_LEVEL,
+ 'called_by': self.TREATMENT_ID
+ }
logger.info(key='rdt_log', data=person_details_for_test)
if dx_result:
@@ -1160,16 +1065,15 @@ def apply(self, person_id, squeeze_factor):
)
# Log the test: line-list of summary information about each test
- fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id=person_id)
- person_details_for_test = _data_for_rdt_log(
- person_id=person_id,
- age=df.at[person_id, 'age_years'],
- fever_is_a_symptom=fever_present,
- dx_result=dx_result,
- facility_level=self.ACCEPTED_FACILITY_LEVEL,
- treatment_id=self.TREATMENT_ID,
- )
-
+ fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id)
+ person_details_for_test = {
+ 'person_id': person_id,
+ 'age': df.at[person_id, 'age_years'],
+ 'fever_present': fever_present,
+ 'rdt_result': dx_result,
+ 'facility_level': self.ACCEPTED_FACILITY_LEVEL,
+ 'called_by': self.TREATMENT_ID
+ }
logger.info(key='rdt_log', data=person_details_for_test)
# if positive, refer for a confirmatory test at level 1a
@@ -1223,15 +1127,15 @@ def apply(self, person_id, squeeze_factor):
# rdt is offered as part of the treatment package
# Log the test: line-list of summary information about each test
- fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id=person_id)
- person_details_for_test = _data_for_rdt_log(
- person_id=person_id,
- age=df.at[person_id, 'age_years'],
- fever_is_a_symptom=fever_present,
- dx_result=True,
- facility_level=self.ACCEPTED_FACILITY_LEVEL,
- treatment_id=self.TREATMENT_ID,
- )
+ fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id)
+ person_details_for_test = {
+ 'person_id': person_id,
+ 'age': df.at[person_id, 'age_years'],
+ 'fever_present': fever_present,
+ 'rdt_result': True,
+ 'facility_level': self.ACCEPTED_FACILITY_LEVEL,
+ 'called_by': self.TREATMENT_ID
+ }
logger.info(key='rdt_log', data=person_details_for_test)
def get_drugs(self, age_of_person):
@@ -1314,21 +1218,17 @@ def apply(self, person_id, squeeze_factor):
df.at[person_id, 'ma_date_tx'] = self.sim.date
df.at[person_id, 'ma_tx_counter'] += 1
- # Add used equipment
- self.add_equipment({'Drip stand', 'Haemoglobinometer',
- 'Analyser, Combined Chemistry and Electrolytes'})
-
# rdt is offered as part of the treatment package
# Log the test: line-list of summary information about each test
- fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id=person_id)
- person_details_for_test = _data_for_rdt_log(
- person_id=person_id,
- age=df.at[person_id, 'age_years'],
- fever_is_a_symptom=fever_present,
- dx_result=True,
- facility_level=self.ACCEPTED_FACILITY_LEVEL,
- treatment_id=self.TREATMENT_ID,
- )
+ fever_present = 'fever' in self.sim.modules["SymptomManager"].has_what(person_id)
+ person_details_for_test = {
+ 'person_id': person_id,
+ 'age': df.at[person_id, 'age_years'],
+ 'fever_present': fever_present,
+ 'rdt_result': True,
+ 'facility_level': self.ACCEPTED_FACILITY_LEVEL,
+ 'called_by': self.TREATMENT_ID
+ }
logger.info(key='rdt_log', data=person_details_for_test)
def did_not_run(self):
@@ -1765,21 +1665,3 @@ def apply(self, population):
logger.info(key='pop_district',
data=pop.to_dict(),
description='District population sizes')
-
-
-def _data_for_rdt_log(
- person_id: int,
- age: int,
- fever_is_a_symptom: bool,
- dx_result: Union[bool, None],
- facility_level: str,
- treatment_id: str,
-):
- return {
- "person_id": person_id,
- "age": age,
- "fever_present": fever_is_a_symptom,
- "rdt_result": pd.array([dx_result], dtype="boolean"),
- "facility_level": facility_level,
- "called_by": treatment_id,
- }
diff --git a/src/tlo/methods/measles.py b/src/tlo/methods/measles.py
index 39f9828860..01481a060c 100644
--- a/src/tlo/methods/measles.py
+++ b/src/tlo/methods/measles.py
@@ -1,28 +1,23 @@
-from __future__ import annotations
-
import math
import os
-from typing import TYPE_CHECKING, List
+from typing import List
import pandas as pd
from tlo import DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.methods import Metadata
from tlo.methods.causes import Cause
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
from tlo.util import random_date
-if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
-
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class Measles(Module, GenericFirstAppointmentsMixin):
+class Measles(Module):
"""This module represents measles infections and disease."""
INIT_DEPENDENCIES = {'Demography', 'HealthSystem', 'SymptomManager'}
@@ -212,14 +207,13 @@ def process_parameters(self):
def do_at_generic_first_appt(
self,
- person_id: int,
+ patient_id: int,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
if "rash" in symptoms:
- event = HSI_Measles_Treatment(person_id=person_id, module=self)
- schedule_hsi_event(event, priority=0, topen=self.sim.date)
+ event = HSI_Measles_Treatment(person_id=patient_id, module=self)
+ self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date)
class MeaslesEvent(RegularEvent, PopulationScopeEventMixin):
@@ -442,7 +436,7 @@ def apply(self, person_id, squeeze_factor):
data=f"HSI_Measles_Treatment: treat person {person_id} for measles")
df = self.sim.population.props
- symptoms = self.sim.modules["SymptomManager"].has_what(person_id=person_id)
+ symptoms = self.sim.modules["SymptomManager"].has_what(person_id)
# for non-complicated measles
item_codes = [self.module.consumables['vit_A']]
@@ -460,9 +454,6 @@ def apply(self, person_id, squeeze_factor):
logger.debug(key="HSI_Measles_Treatment",
data=f"HSI_Measles_Treatment: giving required measles treatment to person {person_id}")
- if "respiratory_symptoms" in symptoms:
- self.add_equipment({'Oxygen concentrator', 'Oxygen cylinder, with regulator'})
-
# modify person property which is checked when scheduled death occurs (or shouldn't occur)
df.at[person_id, "me_on_treatment"] = True
@@ -548,7 +539,7 @@ def apply(self, population):
if tmp:
proportion_with_symptom = number_with_symptom / tmp
else:
- proportion_with_symptom = 0.0
+ proportion_with_symptom = 0
symptom_output[symptom] = proportion_with_symptom
logger.info(key="measles_symptoms",
@@ -586,7 +577,7 @@ def apply(self, population):
if total_infected:
prop_infected_by_age = infected_age_counts / total_infected
else:
- prop_infected_by_age = infected_age_counts.astype("float") # just output the series of zeros by age group
+ prop_infected_by_age = infected_age_counts # just output the series of zeros by age group
logger.info(key='measles_incidence_age_range', data=prop_infected_by_age.to_dict(),
description="measles incidence by age group")
diff --git a/src/tlo/methods/mockitis.py b/src/tlo/methods/mockitis.py
index 6af33c5fc7..e349dd188e 100644
--- a/src/tlo/methods/mockitis.py
+++ b/src/tlo/methods/mockitis.py
@@ -1,26 +1,21 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, List
+from typing import List
import pandas as pd
from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.methods import Metadata
from tlo.methods.causes import Cause
from tlo.methods.demography import InstantaneousDeath
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
-if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
-
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class Mockitis(Module, GenericFirstAppointmentsMixin):
+class Mockitis(Module):
"""This is a dummy infectious disease.
It demonstrates the following behaviours in respect of the healthsystem module:
@@ -295,18 +290,17 @@ def report_daly_values(self):
def do_at_generic_first_appt_emergency(
self,
- person_id: int,
+ patient_id: int,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
# Example for mockitis
if "extreme_pain_in_the_nose" in symptoms:
event = HSI_Mockitis_PresentsForCareWithSevereSymptoms(
module=self,
- person_id=person_id,
+ person_id=patient_id,
)
- schedule_hsi_event(event, priority=1, topen=self.sim.date)
+ self.healthsystem.schedule_hsi_event(event, priority=1, topen=self.sim.date)
class MockitisEvent(RegularEvent, PopulationScopeEventMixin):
"""
diff --git a/src/tlo/methods/newborn_outcomes.py b/src/tlo/methods/newborn_outcomes.py
index 3691bc6003..debfdb3530 100644
--- a/src/tlo/methods/newborn_outcomes.py
+++ b/src/tlo/methods/newborn_outcomes.py
@@ -377,53 +377,43 @@ def get_and_store_newborn_item_codes(self):
This function defines the required consumables for each intervention delivered during this module and stores
them in a module level dictionary called within HSIs
"""
- ic = self.sim.modules['HealthSystem'].get_item_code_from_item_name
+ get_list_of_items = pregnancy_helper_functions.get_list_of_items
- # First we store the item codes for the consumables for which their quantity varies for individuals based on
- # length of pregnancy
- # ---------------------------------- BLOOD TEST EQUIPMENT ---------------------------------------------------
- self.item_codes_nb_consumables['blood_test_equipment'] = \
- {ic('Blood collecting tube, 5 ml'): 1,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1
- }
# ---------------------------------- IV DRUG ADMIN EQUIPMENT -------------------------------------------------
self.item_codes_nb_consumables['iv_drug_equipment'] = \
- {ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1
- }
+ get_list_of_items(self, ['Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box'])
+
+ # ---------------------------------- BLOOD TEST EQUIPMENT ---------------------------------------------------
+ self.item_codes_nb_consumables['blood_test_equipment'] = \
+ get_list_of_items(self, ['Disposables gloves, powder free, 100 pieces per box'])
# -------------------------------------------- VITAMIN K ------------------------------------------
self.item_codes_nb_consumables['vitamin_k'] = \
- {ic('vitamin K1 (phytomenadione) 1 mg/ml, 1 ml, inj._100_IDA'): 1}
+ get_list_of_items(self, ['vitamin K1 (phytomenadione) 1 mg/ml, 1 ml, inj._100_IDA'])
# -------------------------------------------- EYE CARE ------------------------------------------
- self.item_codes_nb_consumables['eye_care'] = \
- {ic('Tetracycline eye ointment, 1 %, tube 5 mg'): 5}
+ self.item_codes_nb_consumables['eye_care'] = get_list_of_items(
+ self, ['Tetracycline eye ointment, 1 %, tube 5 mg'])
# ------------------------------------- SEPSIS - FULL SUPPORTIVE CARE ---------------------------------------
- # Whilst abx for newborns are weight based the maximum dose does not exceed the minimum unit for the costing
- # model
self.item_codes_nb_consumables['sepsis_supportive_care_core'] = \
- {ic('Benzylpenicillin 1g (1MU), PFR_Each_CMST'): 1,
- ic('Gentamicin 40mg/ml, 2ml_each_CMST'): 1,
- ic('Oxygen, 1000 liters, primarily with oxygen cylinders'): 5760 #
- }
+ get_list_of_items(self, ['Benzylpenicillin 1g (1MU), PFR_Each_CMST',
+ 'Gentamicin 40mg/ml, 2ml_each_CMST',
+ 'Oxygen, 1000 liters, primarily with oxygen cylinders'])
self.item_codes_nb_consumables['sepsis_supportive_care_optional'] = \
- {ic('Dextrose (glucose) 5%, 1000ml_each_CMST'): 500,
- ic('Tube, feeding CH 8_each_CMST'): 1,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Disposables gloves, powder free, 100 pieces per box'): 1
- }
+ get_list_of_items(self, ['Dextrose (glucose) 5%, 1000ml_each_CMST',
+ 'Tube, feeding CH 8_each_CMST',
+ 'Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Giving set iv administration + needle 15 drops/ml_each_CMST',
+ 'Disposables gloves, powder free, 100 pieces per box'])
# ---------------------------------------- SEPSIS - ANTIBIOTICS ---------------------------------------------
- self.item_codes_nb_consumables['sepsis_abx'] = \
- {ic('Benzylpenicillin 1g (1MU), PFR_Each_CMST'): 1,
- ic('Gentamicin 40mg/ml, 2ml_each_CMST'): 1,
- }
+ self.item_codes_nb_consumables['sepsis_abx'] =\
+ get_list_of_items(self, ['Benzylpenicillin 1g (1MU), PFR_Each_CMST',
+ 'Gentamicin 40mg/ml, 2ml_each_CMST'])
def initialise_simulation(self, sim):
# For the first period (2010-2015) we use the first value in each list as a parameter
@@ -979,27 +969,23 @@ def assessment_and_treatment_newborn_sepsis(self, hsi_event, facility_type):
# check consumables
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_nb_consumables['sepsis_supportive_care_core'],
- opt_cons=self.item_codes_nb_consumables['sepsis_supportive_care_optional'])
+ self, hsi_event, self.item_codes_nb_consumables, core='sepsis_supportive_care_core',
+ optional='sepsis_supportive_care_optional')
# Then, if the consumables are available, treatment for sepsis is delivered
if avail and sf_check:
df.at[person_id, 'nb_supp_care_neonatal_sepsis'] = True
pregnancy_helper_functions.log_met_need(self, 'neo_sep_supportive_care', hsi_event)
- hsi_event.add_equipment({'Drip stand', 'Infusion pump'})
# The same pattern is then followed for health centre care
else:
avail = pregnancy_helper_functions.return_cons_avail(
- self, hsi_event,
- cons=self.item_codes_nb_consumables['sepsis_abx'],
- opt_cons=self.item_codes_nb_consumables['iv_drug_equipment'])
+ self, hsi_event, self.item_codes_nb_consumables, core='sepsis_abx',
+ optional='iv_drug_equipment')
if avail and sf_check:
df.at[person_id, 'nb_inj_abx_neonatal_sepsis'] = True
pregnancy_helper_functions.log_met_need(self, 'neo_sep_abx', hsi_event)
- hsi_event.add_equipment({'Drip stand', 'Infusion pump', 'Oxygen cylinder, with regulator'})
def link_twins(self, child_one, child_two, mother_id):
"""
@@ -1363,7 +1349,7 @@ def apply(self, person_id, squeeze_factor):
# Log the PNC check
logger.info(key='postnatal_check', data={'person_id': person_id,
- 'delivery_setting': str(nci[person_id]['delivery_setting']),
+ 'delivery_setting': nci[person_id]['delivery_setting'],
'visit_number': df.at[person_id, 'nb_pnc_check'],
'timing': nci[person_id]['will_receive_pnc']})
diff --git a/src/tlo/methods/oesophagealcancer.py b/src/tlo/methods/oesophagealcancer.py
index 8adc0614e1..b3a302bcd9 100644
--- a/src/tlo/methods/oesophagealcancer.py
+++ b/src/tlo/methods/oesophagealcancer.py
@@ -14,6 +14,7 @@
import pandas as pd
from tlo import DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata
@@ -22,18 +23,16 @@
from tlo.methods.demography import InstantaneousDeath
from tlo.methods.dxmanager import DxTest
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class OesophagealCancer(Module, GenericFirstAppointmentsMixin):
+class OesophagealCancer(Module):
"""Oesophageal Cancer Disease Module"""
def __init__(self, name=None, resourcefilepath=None):
@@ -579,19 +578,18 @@ def report_daly_values(self):
def do_at_generic_first_appt(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
# If the symptoms include dysphagia, and the patient is not a child,
# begin investigation for Oesophageal Cancer:
- if individual_properties["age_years"] > 5 and "dysphagia" in symptoms:
+ if patient_details.age_years > 5 and "dysphagia" in symptoms:
event = HSI_OesophagealCancer_Investigation_Following_Dysphagia(
- person_id=person_id, module=self
+ person_id=patient_id, module=self
)
- schedule_hsi_event(event, priority=0, topen=self.sim.date)
+ self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date)
# ---------------------------------------------------------------------------------------------------------
@@ -681,22 +679,20 @@ def apply(self, person_id, squeeze_factor):
return hs.get_blank_appt_footprint()
# Check that this event has been called for someone with the symptom dysphagia
- assert 'dysphagia' in self.sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert 'dysphagia' in self.sim.modules['SymptomManager'].has_what(person_id)
# If the person is already diagnosed, then take no action:
if not pd.isnull(df.at[person_id, "oc_date_diagnosis"]):
return hs.get_blank_appt_footprint()
# Check the consumables are available
- cons_avail = self.get_consumables(item_codes=self.module.item_codes_oesophageal_can['screening_endoscopy_core'],
+ # todo: replace with endoscope?
+ cons_avail = self.get_consumables(item_codes=self.module.item_codes_oesophageal_can['screening_biopsy_core'],
optional_item_codes=
- self.module.item_codes_oesophageal_can[
- 'screening_biopsy_endoscopy_cystoscopy_optional'])
+ self.module.item_codes_oesophageal_can['screening_biopsy_optional'])
if cons_avail:
- # If consumables are available add used equipment and run the dx_test representing the biopsy
- # n.b. endoscope not in equipment list
- self.add_equipment({'Endoscope', 'Ordinary Microscope'})
+ # If consumables are available, run the dx_test representing the biopsy
# Use an endoscope to diagnose whether the person has Oesophageal Cancer:
dx_result = hs.dx_manager.run_dx_test(
@@ -785,8 +781,7 @@ def apply(self, person_id, squeeze_factor):
self.module.item_codes_oesophageal_can['treatment_surgery_optional'])
if cons_avail:
- # If consumables are available and the treatment will go ahead - update the equipment
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery'))
+ # If consumables are available and the treatment will go ahead
# Log chemotherapy consumables
self.get_consumables(
@@ -895,8 +890,7 @@ def apply(self, person_id, squeeze_factor):
item_codes=self.module.item_codes_oesophageal_can['palliation'])
if cons_available:
- # If consumables are available and the treatment will go ahead - update the equipment
- self.add_equipment({'Infusion pump', 'Drip stand'})
+ # If consumables are available and the treatment will go ahead
# Record the start of palliative care if this is first appointment
if pd.isnull(df.at[person_id, "oc_date_palliative_care"]):
diff --git a/src/tlo/methods/other_adult_cancers.py b/src/tlo/methods/other_adult_cancers.py
index 5aad8f971a..3bea8933e6 100644
--- a/src/tlo/methods/other_adult_cancers.py
+++ b/src/tlo/methods/other_adult_cancers.py
@@ -12,6 +12,7 @@
import pandas as pd
from tlo import DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata
@@ -20,18 +21,16 @@
from tlo.methods.demography import InstantaneousDeath
from tlo.methods.dxmanager import DxTest
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class OtherAdultCancer(Module, GenericFirstAppointmentsMixin):
+class OtherAdultCancer(Module):
"""Other Adult Cancers Disease Module"""
def __init__(self, name=None, resourcefilepath=None):
@@ -577,18 +576,17 @@ def report_daly_values(self):
def do_at_generic_first_appt(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs
- ) -> None:
- if individual_properties["age_years"] > 5 and "early_other_adult_ca_symptom" in symptoms:
+ ) -> IndividualPropertyUpdates:
+ if patient_details.age_years > 5 and "early_other_adult_ca_symptom" in symptoms:
event = HSI_OtherAdultCancer_Investigation_Following_early_other_adult_ca_symptom(
- person_id=person_id,
+ person_id=patient_id,
module=self,
)
- schedule_hsi_event(event, priority=0, topen=self.sim.date)
+ self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date)
# ---------------------------------------------------------------------------------------------------------
@@ -685,7 +683,7 @@ def apply(self, person_id, squeeze_factor):
return hs.get_blank_appt_footprint()
# Check that this event has been called for someone with the symptom other_adult_ca_symptom
- assert 'early_other_adult_ca_symptom' in self.sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert 'early_other_adult_ca_symptom' in self.sim.modules['SymptomManager'].has_what(person_id)
# If the person is already diagnosed, then take no action:
if not pd.isnull(df.at[person_id, "oac_date_diagnosis"]):
@@ -694,12 +692,10 @@ def apply(self, person_id, squeeze_factor):
# Check consumables are available
cons_avail = self.get_consumables(item_codes=self.module.item_codes_other_can['screening_biopsy_core'],
optional_item_codes=
- self.module.item_codes_other_can[
- 'screening_biopsy_endoscopy_cystoscopy_optional'])
+ self.module.item_codes_other_can['screening_biopsy_optional'])
if cons_avail:
- # If consumables are available add used equipment and run the dx_test representing the biopsy
- self.add_equipment({'Ultrasound scanning machine', 'Ordinary Microscope'})
+ # If consumables are available, run the dx_test representing the biopsy
# Use a diagnostic_device to diagnose whether the person has other adult cancer:
dx_result = hs.dx_manager.run_dx_test(
@@ -788,8 +784,7 @@ def apply(self, person_id, squeeze_factor):
)
if cons_available:
- # If consumables are available and the treatment will go ahead - update the equipment
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery'))
+ # If consumables are available and the treatment will go ahead
# Record date and stage of starting treatment
df.at[person_id, "oac_date_treatment"] = self.sim.date
@@ -900,8 +895,7 @@ def apply(self, person_id, squeeze_factor):
item_codes=self.module.item_codes_other_can['palliation'])
if cons_available:
- # If consumables are available and the treatment will go ahead - update the equipment
- self.add_equipment({'Infusion pump', 'Drip stand'})
+ # If consumables are available and the treatment will go ahead
# Record the start of palliative care if this is first appointment
if pd.isnull(df.at[person_id, "oac_date_palliative_care"]):
diff --git a/src/tlo/methods/postnatal_supervisor.py b/src/tlo/methods/postnatal_supervisor.py
index 25bce6013f..25a2d54f6d 100644
--- a/src/tlo/methods/postnatal_supervisor.py
+++ b/src/tlo/methods/postnatal_supervisor.py
@@ -1274,28 +1274,24 @@ def apply(self, person_id, squeeze_factor):
return
# Define the consumables
- ic = self.sim.modules['HealthSystem'].get_item_code_from_item_name
-
- of_repair_cons = \
- {ic('Scalpel blade size 22 (individually wrapped)_100_CMST'): 1,
- ic('Halothane (fluothane)_250ml_CMST'): 100,
- ic('Ceftriaxone 1g, PFR_each_CMST'): 2,
- ic('Metronidazole 200mg_1000_CMST'): 6000,
- ic('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- ic('Paracetamol, tablet, 500 mg'): 8000,
- ic('Declofenac injection_each_CMST'): 1,
- ic('Foley catheter'): 1,
- ic('Bag, urine, collecting, 2000 ml'): 1,
- ic("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 2000,
- ic('Sodium chloride, injectable solution, 0,9 %, 500 ml'): 2000,
- ic('Giving set iv administration + needle 15 drops/ml_each_CMST'): 1,
- ic('Chlorhexidine 1.5% solution_5_CMST'): 50,
- }
+ of_repair_cons = pregnancy_helper_functions.get_list_of_items(
+ self, ['Scalpel blade size 22 (individually wrapped)_100_CMST',
+ 'Halothane (fluothane)_250ml_CMST',
+ 'Ceftriaxone 1g, PFR_each_CMST',
+ 'Metronidazole 200mg_1000_CMST',
+ 'Cannula iv (winged with injection pot) 18_each_CMST',
+ 'Paracetamol, tablet, 500 mg',
+ 'Declofenac injection_each_CMST',
+ 'Pethidine, 50 mg/ml, 2 ml ampoule',
+ 'Foley catheter',
+ 'Bag, urine, collecting, 2000 ml',
+ "ringer's lactate (Hartmann's solution), 1000 ml_12_IDA",
+ 'Sodium chloride, injectable solution, 0,9 %, 500 ml',
+ "Giving set iv administration + needle 15 drops/ml_each_CMST",
+ "Chlorhexidine 1.5% solution_5_CMST"])
self.get_consumables(item_codes=of_repair_cons)
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery'))
-
# Log the end of disability in the MNI
pregnancy_helper_functions.store_dalys_in_mni(
person_id, self.sim.modules['PregnancySupervisor'].mother_and_newborn_info,
diff --git a/src/tlo/methods/pregnancy_helper_functions.py b/src/tlo/methods/pregnancy_helper_functions.py
index 8f7faa0503..20a712f134 100644
--- a/src/tlo/methods/pregnancy_helper_functions.py
+++ b/src/tlo/methods/pregnancy_helper_functions.py
@@ -21,7 +21,7 @@ def get_list_of_items(self, item_list):
return codes
-def return_cons_avail(self, hsi_event, cons, opt_cons):
+def return_cons_avail(self, hsi_event, cons_dict, **info):
"""
This function is called by majority of interventions across maternal and neonatal modules to return whether a
consumable or package of consumables are available. If analysis is not being conducted (as indicated by a series of
@@ -38,12 +38,21 @@ def return_cons_avail(self, hsi_event, cons, opt_cons):
ps_params = self.sim.modules['PregnancySupervisor'].current_parameters
la_params = self.sim.modules['Labour'].current_parameters
- if opt_cons is None:
+ # If 'number' is passed as an optional argument then a predetermined number of consumables will be requested
+ if 'number' in info.keys():
+ core_cons = {cons_dict[info['core']][0]: info['number']}
+ else:
+ core_cons = cons_dict[info['core']]
+
+ # If 'optional' is passed then the optional set of consumables is selected from the consumables dict
+ if 'optional' in info.keys():
+ opt_cons = cons_dict[info['optional']]
+ else:
opt_cons = []
# Check if analysis is currently running, if not then availability is determined normally
if not ps_params['ps_analysis_in_progress'] and not la_params['la_analysis_in_progress']:
- available = hsi_event.get_consumables(item_codes=cons,
+ available = hsi_event.get_consumables(item_codes=core_cons,
optional_item_codes=opt_cons)
if not available and (hsi_event.target in mni) and (hsi_event != 'AntenatalCare_Outpatient'):
@@ -52,7 +61,7 @@ def return_cons_avail(self, hsi_event, cons, opt_cons):
return available
else:
- available = hsi_event.get_consumables(item_codes=cons, optional_item_codes=opt_cons)
+ available = hsi_event.get_consumables(item_codes=core_cons, optional_item_codes=opt_cons)
# Depending on HSI calling this function a different parameter set is used to determine if analysis is being
# conducted
diff --git a/src/tlo/methods/pregnancy_supervisor.py b/src/tlo/methods/pregnancy_supervisor.py
index 7dd8819ab6..7d89181344 100644
--- a/src/tlo/methods/pregnancy_supervisor.py
+++ b/src/tlo/methods/pregnancy_supervisor.py
@@ -18,6 +18,7 @@
logging,
util,
)
+from tlo.core import IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel
from tlo.methods import Metadata, labour, pregnancy_helper_functions, pregnancy_supervisor_lm
@@ -26,18 +27,16 @@
HSI_CareOfWomenDuringPregnancy_TreatmentForEctopicPregnancy,
)
from tlo.methods.causes import Cause
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.util import BitsetHandler
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class PregnancySupervisor(Module, GenericFirstAppointmentsMixin):
+class PregnancySupervisor(Module):
"""This module is responsible for simulating the antenatal period of pregnancy (the period from conception until
the termination of pregnancy). A number of outcomes are managed by this module including early pregnancy loss
(induced/spontaneous abortion, ectopic pregnancy and antenatal stillbirth) and pregnancy complications of the
@@ -1674,11 +1673,10 @@ def schedule_late_visit(df_slice):
def do_at_generic_first_appt_emergency(
self,
- person_id: int,
- individual_properties: IndividualProperties,
- schedule_hsi_event: HSIEventScheduler,
+ patient_id: int,
+ patient_details: PatientDetails,
**kwargs,
- ) -> None:
+ ) -> IndividualPropertyUpdates:
scheduling_options = {
"priority": 0,
"topen": self.sim.date,
@@ -1686,25 +1684,25 @@ def do_at_generic_first_appt_emergency(
}
# ----- ECTOPIC PREGNANCY -----
- if individual_properties["ps_ectopic_pregnancy"] != 'none':
+ if patient_details.ps_ectopic_pregnancy != 'none':
event = HSI_CareOfWomenDuringPregnancy_TreatmentForEctopicPregnancy(
module=self.sim.modules["CareOfWomenDuringPregnancy"],
- person_id=person_id,
+ person_id=patient_id,
)
- schedule_hsi_event(event, **scheduling_options)
+ self.healthsystem.schedule_hsi_event(event, **scheduling_options)
# ----- COMPLICATIONS OF ABORTION -----
abortion_complications = self.sim.modules[
"PregnancySupervisor"
].abortion_complications
if abortion_complications.has_any(
- [person_id], "sepsis", "injury", "haemorrhage", first=True
+ [patient_id], "sepsis", "injury", "haemorrhage", first=True
):
event = HSI_CareOfWomenDuringPregnancy_PostAbortionCaseManagement(
module=self.sim.modules["CareOfWomenDuringPregnancy"],
- person_id=person_id,
+ person_id=patient_id,
)
- schedule_hsi_event(event, **scheduling_options)
+ self.healthsystem.schedule_hsi_event(event, **scheduling_options)
class PregnancySupervisorEvent(RegularEvent, PopulationScopeEventMixin):
""" This is the PregnancySupervisorEvent, it is a weekly event which has four primary functions.
diff --git a/src/tlo/methods/prostate_cancer.py b/src/tlo/methods/prostate_cancer.py
index dbbe2c427f..f9520052b1 100644
--- a/src/tlo/methods/prostate_cancer.py
+++ b/src/tlo/methods/prostate_cancer.py
@@ -12,6 +12,7 @@
import pandas as pd
from tlo import DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata
@@ -20,18 +21,16 @@
from tlo.methods.demography import InstantaneousDeath
from tlo.methods.dxmanager import DxTest
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class ProstateCancer(Module, GenericFirstAppointmentsMixin):
+class ProstateCancer(Module):
"""Prostate Cancer Disease Module"""
def __init__(self, name=None, resourcefilepath=None):
@@ -591,30 +590,29 @@ def report_daly_values(self):
def do_at_generic_first_appt(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs
- ) -> None:
+ ) -> IndividualPropertyUpdates:
# If the patient is not a child, and symptoms are indicative,
# begin investigation for prostate cancer
scheduling_options = {
"priority": 0,
"topen": self.sim.date,
}
- if individual_properties["age_years"] > 5:
+ if patient_details.age_years > 5:
if "urinary" in symptoms:
event = HSI_ProstateCancer_Investigation_Following_Urinary_Symptoms(
- person_id=person_id, module=self
+ person_id=patient_id, module=self
)
- schedule_hsi_event(event, **scheduling_options)
+ self.healthsystem.schedule_hsi_event(event, **scheduling_options)
if "pelvic_pain" in symptoms:
event = HSI_ProstateCancer_Investigation_Following_Pelvic_Pain(
- person_id=person_id, module=self
+ person_id=patient_id, module=self
)
- schedule_hsi_event(event, **scheduling_options)
+ self.healthsystem.schedule_hsi_event(event, **scheduling_options)
# ---------------------------------------------------------------------------------------------------------
@@ -719,7 +717,7 @@ def apply(self, person_id, squeeze_factor):
return hs.get_blank_appt_footprint()
# Check that this event has been called for someone with the urinary symptoms
- assert 'urinary' in self.sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert 'urinary' in self.sim.modules['SymptomManager'].has_what(person_id)
# If the person is already diagnosed, then take no action:
if not pd.isnull(df.at[person_id, "pc_date_diagnosis"]):
@@ -767,7 +765,7 @@ def apply(self, person_id, squeeze_factor):
return hs.get_blank_appt_footprint()
# Check that this event has been called for someone with the pelvic pain
- assert 'pelvic_pain' in self.sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert 'pelvic_pain' in self.sim.modules['SymptomManager'].has_what(person_id)
# If the person is already diagnosed, then take no action:
if not pd.isnull(df.at[person_id, "pc_date_diagnosis"]):
@@ -782,6 +780,7 @@ def apply(self, person_id, squeeze_factor):
hsi_event=self
)
+ # TODO: replace with PSA test when added to cons list
cons_avail = self.get_consumables(item_codes=self.module.item_codes_prostate_can['screening_psa_test_optional'])
if dx_result and cons_avail:
@@ -823,12 +822,9 @@ def apply(self, person_id, squeeze_factor):
cons_available = self.get_consumables(item_codes=self.module.item_codes_prostate_can['screening_biopsy_core'],
optional_item_codes=self.module.item_codes_prostate_can[
- 'screening_biopsy_endoscopy_cystoscopy_optional'])
+ 'screening_biopsy_optional'])
if cons_available:
- # If consumables are available update the use of equipment and run the dx_test representing the biopsy
- self.add_equipment({'Ultrasound scanning machine', 'Ordinary Microscope'})
-
# Use a biopsy to assess whether the person has prostate cancer:
dx_result = hs.dx_manager.run_dx_test(
dx_tests_to_run='biopsy_for_prostate_cancer',
@@ -916,8 +912,7 @@ def apply(self, person_id, squeeze_factor):
'treatment_surgery_optional'])
if cons_available:
- # If consumables are available and the treatment will go ahead - update the equipment
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery'))
+ # If consumables are available the treatment will go ahead
# Record date and stage of starting treatment
df.at[person_id, "pc_date_treatment"] = self.sim.date
@@ -1021,8 +1016,7 @@ def apply(self, person_id, squeeze_factor):
item_codes=self.module.item_codes_prostate_can['palliation'])
if cons_available:
- # If consumables are available and the treatment will go ahead - update the equipment
- self.add_equipment({'Infusion pump', 'Drip stand'})
+ # If consumables are available and the treatment will go ahead
# Record the start of palliative care if this is first appointment
if pd.isnull(df.at[person_id, "pc_date_palliative_care"]):
diff --git a/src/tlo/methods/rti.py b/src/tlo/methods/rti.py
index 68ef59fcf0..654378c4bf 100644
--- a/src/tlo/methods/rti.py
+++ b/src/tlo/methods/rti.py
@@ -11,17 +11,16 @@
import pandas as pd
from tlo import DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata
from tlo.methods.causes import Cause
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
# ---------------------------------------------------------------------------------------------------------
# MODULE DEFINITIONS
@@ -31,7 +30,7 @@
logger.setLevel(logging.DEBUG)
-class RTI(Module, GenericFirstAppointmentsMixin):
+class RTI(Module):
"""
The road traffic injuries module for the TLO model, handling all injuries related to road traffic accidents.
"""
@@ -41,7 +40,7 @@ def __init__(self, name=None, resourcefilepath=None):
super().__init__(name)
self.resourcefilepath = resourcefilepath
self.ASSIGN_INJURIES_AND_DALY_CHANGES = None
- self.cons_item_codes = None # (Will store consumable item codes)
+ self.item_codes_for_consumables_required = dict()
INIT_DEPENDENCIES = {"SymptomManager",
"HealthBurden"}
@@ -1016,10 +1015,6 @@ def __init__(self, name=None, resourcefilepath=None):
Types.INT,
"A cut-off score above which an injuries will be considered severe enough to cause mortality in those who"
"have not sought care."
- ),
- 'maximum_number_of_times_HSI_events_should_run': Parameter(
- Types.INT,
- "limit on the number of times an HSI event can run"
)
}
@@ -1529,8 +1524,6 @@ def initialise_simulation(self, sim):
sim.schedule_event(RTI_Check_Death_No_Med(self), sim.date + DateOffset(months=0))
# Begin logging the RTI events
sim.schedule_event(RTI_Logging_Event(self), sim.date + DateOffset(months=1))
- # Look-up consumable item codes
- self.look_up_consumable_item_codes()
def rti_do_when_diagnosed(self, person_id):
"""
@@ -2297,129 +2290,6 @@ def on_birth(self, mother_id, child_id):
df.at[child_id, 'rt_debugging_DALY_wt'] = 0
df.at[child_id, 'rt_injuries_left_untreated'] = []
- def look_up_consumable_item_codes(self):
- """Look up the item codes that used in the HSI in the module"""
- get_item_codes = self.sim.modules['HealthSystem'].get_item_code_from_item_name
-
- self.cons_item_codes = dict()
- self.cons_item_codes['shock_treatment_child'] = {
- get_item_codes("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 500,
- get_item_codes("Dextrose (glucose) 5%, 1000ml_each_CMST"): 500,
- get_item_codes('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- get_item_codes('Blood, one unit'): 2,
- get_item_codes("Oxygen, 1000 liters, primarily with oxygen cylinders"): 23_040
- }
- self.cons_item_codes['shock_treatment_adult'] = {
- get_item_codes("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 2000,
- get_item_codes('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- get_item_codes('Blood, one unit'): 2,
- get_item_codes("Oxygen, 1000 liters, primarily with oxygen cylinders"): 23_040
- }
- self.cons_item_codes['fracture_treatment_plaster'] = {
- get_item_codes('Plaster of Paris (POP) 10cm x 7.5cm slab_12_CMST'): 1
- # This is for one fracture.
- }
- self.cons_item_codes['fracture_treatment_bandage'] = {
- get_item_codes('Bandage, crepe 7.5cm x 1.4m long , when stretched'): 200,
- # (The 200 is a standard assumption for the amount of bandage needed, irrespective of the number of
- # fractures.)
- }
- self.cons_item_codes['open_fracture_treatment'] = {
- get_item_codes('Ceftriaxone 1g, PFR_each_CMST'): 2,
- get_item_codes('Cetrimide 15% + chlorhexidine 1.5% solution.for dilution _5_CMST'): 100,
- get_item_codes("Gauze, absorbent 90cm x 40m_each_CMST"): 100,
- get_item_codes('Suture pack'): 1,
- }
- self.cons_item_codes["open_fracture_treatment_additional_if_contaminated"] = {
- get_item_codes('Metronidazole, injection, 500 mg in 100 ml vial'): 3
- }
-
- self.cons_item_codes['laceration_treatment_suture_pack'] = {
- get_item_codes('Suture pack'): 1,
- }
- self.cons_item_codes['laceration_treatment_cetrimide_chlorhexidine'] = {
- get_item_codes('Cetrimide 15% + chlorhexidine 1.5% solution.for dilution _5_CMST'): 100,
- }
- self.cons_item_codes['burn_treatment_per_burn'] = {
- get_item_codes("Gauze, absorbent 90cm x 40m_each_CMST"): 100,
- get_item_codes('Cetrimide 15% + chlorhexidine 1.5% solution.for dilution _5_CMST'): 100,
- }
- self.cons_item_codes['ringers lactate for multiple burns'] = {
- get_item_codes("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 4000
- }
- self.cons_item_codes['tetanus_treatment'] = {get_item_codes('Tetanus toxoid, injection'): 1}
- self.cons_item_codes['pain_management_mild_under_16'] = {get_item_codes("Paracetamol 500mg_1000_CMST"): 8000}
- self.cons_item_codes['pain_management_mild_above_16'] = {
- get_item_codes("diclofenac sodium 25 mg, enteric coated_1000_IDA"): 300
- }
- self.cons_item_codes['pain_management_moderate'] = {
- get_item_codes("tramadol HCl 100 mg/2 ml, for injection_100_IDA"): 3
- }
- self.cons_item_codes['pain_management_severe'] = {
- get_item_codes("morphine sulphate 10 mg/ml, 1 ml, injection (nt)_10_IDA"): 12
- }
- self.cons_item_codes['major_surgery'] = {
- # request a general anaesthetic
- get_item_codes("Halothane (fluothane)_250ml_CMST"): 100,
- # clean the site of the surgery
- get_item_codes("Chlorhexidine 1.5% solution_5_CMST"): 600,
- # tools to begin surgery
- get_item_codes("Scalpel blade size 22 (individually wrapped)_100_CMST"): 1,
- # administer an IV
- get_item_codes('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- get_item_codes("Giving set iv administration + needle 15 drops/ml_each_CMST"): 1,
- get_item_codes("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 2000,
- # repair incision made
- get_item_codes("Suture pack"): 1,
- get_item_codes("Gauze, absorbent 90cm x 40m_each_CMST"): 100,
- # administer pain killer
- get_item_codes('Pethidine, 50 mg/ml, 2 ml ampoule'): 6,
- # administer antibiotic
- get_item_codes("Ampicillin injection 500mg, PFR_each_CMST"): 2,
- # equipment used by surgeon, gloves and facemask
- get_item_codes('Disposables gloves, powder free, 100 pieces per box'): 1,
- get_item_codes('surgical face mask, disp., with metal nose piece_50_IDA'): 1,
- # request syringe
- get_item_codes("Syringe, Autodisable SoloShot IX "): 1
- }
- self.cons_item_codes['minor_surgery'] = {
- # request a local anaesthetic
- get_item_codes("Halothane (fluothane)_250ml_CMST"): 100,
- # clean the site of the surgery
- get_item_codes("Chlorhexidine 1.5% solution_5_CMST"): 300,
- # tools to begin surgery
- get_item_codes("Scalpel blade size 22 (individually wrapped)_100_CMST"): 1,
- # administer an IV
- get_item_codes('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
- get_item_codes("Giving set iv administration + needle 15 drops/ml_each_CMST"): 1,
- get_item_codes("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 2000,
- # repair incision made
- get_item_codes("Suture pack"): 1,
- get_item_codes("Gauze, absorbent 90cm x 40m_each_CMST"): 100,
- # administer pain killer
- get_item_codes('Pethidine, 50 mg/ml, 2 ml ampoule'): 6,
- # administer antibiotic
- get_item_codes("Ampicillin injection 500mg, PFR_each_CMST"): 2,
- # equipment used by surgeon, gloves and facemask
- get_item_codes('Disposables gloves, powder free, 100 pieces per box'): 1,
- get_item_codes('surgical face mask, disp., with metal nose piece_50_IDA'): 1,
- # request syringe
- get_item_codes("Syringe, Autodisable SoloShot IX "): 1
- }
- # Function to get the consumables for fracture treatment, which depends on the number of fractures:
- self.cons_item_codes['fracture_treatment'] = lambda num_fractures: {
- **{item: num_fractures for item in self.cons_item_codes['fracture_treatment_plaster']},
- **self.cons_item_codes['fracture_treatment_bandage']
- }
- # Function to get the consumables for laceration treatment, which depends on the number of lacerations:
- self.cons_item_codes['laceration_treatment'] = lambda num_laceration: {
- **{item: num_laceration for item in self.cons_item_codes['laceration_treatment_suture_pack']},
- **self.cons_item_codes['laceration_treatment_cetrimide_chlorhexidine']
- }
- self.cons_item_codes['burn_treatment'] = lambda num_burns: {
- item: num_burns for item in self.cons_item_codes['burn_treatment_per_burn']
- }
-
def on_hsi_alert(self, person_id, treatment_id):
"""
This is called whenever there is an HSI event commissioned by one of the other disease modules.
@@ -2572,7 +2442,7 @@ def rti_assign_injuries(self, number):
inc_other = other_counts / ((n_alive - other_counts) * 1 / 12) * 100000
tot_inc_all_inj = inc_amputations + inc_burns + inc_fractures + inc_tbi + inc_sci + inc_minor + inc_other
if number > 0:
- number_of_injuries = int(inj_df['Number_of_injuries'].iloc[0])
+ number_of_injuries = inj_df['Number_of_injuries'].tolist()
else:
number_of_injuries = 0
dict_to_output = {'inc_amputations': inc_amputations,
@@ -2614,7 +2484,7 @@ def rti_assign_injuries(self, number):
if n_lx_fracs > 0:
proportion_lx_fracture_open = n_open_lx_fracs / n_lx_fracs
else:
- proportion_lx_fracture_open = float("nan")
+ proportion_lx_fracture_open = 'no_lx_fractures'
injury_info = {'Proportion_lx_fracture_open': proportion_lx_fracture_open}
logger.info(key='Open_fracture_information',
data=injury_info,
@@ -2624,33 +2494,33 @@ def rti_assign_injuries(self, number):
def _common_first_appt_steps(
self,
- person_id: int,
- individual_properties: IndividualProperties,
- schedule_hsi_event: HSIEventScheduler,
- ) -> None:
+ patient_id: int,
+ patient_details: PatientDetails,
+ ) -> IndividualPropertyUpdates:
"""
Shared logic steps that are used by the RTI module when a generic HSI
event is to be scheduled.
"""
+ patient_details_updates = {}
# Things to do upon a person presenting at a Non-Emergency Generic
# HSI if they have an injury.
persons_injuries = [
- individual_properties[injury] for injury in RTI.INJURY_COLUMNS
+ getattr(patient_details, injury) for injury in RTI.INJURY_COLUMNS
]
if (
- pd.isnull(individual_properties["cause_of_death"])
- and not individual_properties["rt_diagnosed"]
+ pd.isnull(patient_details.cause_of_death)
+ and not patient_details.rt_diagnosed
):
if set(RTI.INJURIES_REQ_IMAGING).intersection(set(persons_injuries)):
- if individual_properties["is_alive"]:
- event = HSI_RTI_Imaging_Event(module=self, person_id=person_id)
- schedule_hsi_event(
+ if patient_details.is_alive:
+ event = HSI_RTI_Imaging_Event(module=self, person_id=patient_id)
+ self.healthsystem.schedule_hsi_event(
event,
priority=0,
topen=self.sim.date + DateOffset(days=1),
tclose=self.sim.date + DateOffset(days=15),
)
- individual_properties["rt_diagnosed"] = True
+ patient_details_updates["rt_diagnosed"] = True
# The injured person has been diagnosed in A&E and needs to progress further
# through the health system.
@@ -2676,56 +2546,47 @@ def _common_first_appt_steps(
# If they meet the requirements, send them to HSI_RTI_MedicalIntervention for further treatment
# Using counts condition to stop spurious symptoms progressing people through the model
if counts > 0:
- event = HSI_RTI_Medical_Intervention(module=self, person_id=person_id)
- schedule_hsi_event(
- event,
- priority=0,
- topen=self.sim.date,
+ event = HSI_RTI_Medical_Intervention(module=self, person_id=patient_id)
+ self.healthsystem.schedule_hsi_event(
+ event, priority=0, topen=self.sim.date,
)
# We now check if they need shock treatment
- if (
- individual_properties["rt_in_shock"]
- and individual_properties["is_alive"]
- ):
- event = HSI_RTI_Shock_Treatment(module=self, person_id=person_id)
- schedule_hsi_event(
+ if patient_details.rt_in_shock and patient_details.is_alive:
+ event = HSI_RTI_Shock_Treatment(module=self, person_id=patient_id)
+ self.healthsystem.schedule_hsi_event(
event,
priority=0,
topen=self.sim.date + DateOffset(days=1),
tclose=self.sim.date + DateOffset(days=15),
)
+ return patient_details_updates
def do_at_generic_first_appt(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs
- ) -> None:
+ ) -> IndividualPropertyUpdates:
if "injury" in symptoms:
return self._common_first_appt_steps(
- person_id=person_id,
- individual_properties=individual_properties,
- schedule_hsi_event=schedule_hsi_event,
+ patient_id=patient_id, patient_details=patient_details
)
def do_at_generic_first_appt_emergency(
self,
- person_id: int,
- individual_properties: IndividualProperties,
+ patient_id: int,
+ patient_details: PatientDetails,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs
- ) -> None:
+ ) -> IndividualPropertyUpdates:
# Same process is followed for emergency and non emergency appointments, except the
# initial symptom check
if "severe_trauma" in symptoms:
return self._common_first_appt_steps(
- person_id=person_id,
- individual_properties=individual_properties,
- schedule_hsi_event=schedule_hsi_event,
+ patient_id=patient_id,
+ patient_details=patient_details
)
@@ -2939,7 +2800,7 @@ def apply(self, population):
df.loc[shock_index, 'rt_in_shock'] = True
# log the percentage of those with RTIs in shock
percent_in_shock = \
- len(shock_index) / len(selected_for_rti_inj) if len(selected_for_rti_inj) > 0 else float("nan")
+ len(shock_index) / len(selected_for_rti_inj) if len(selected_for_rti_inj) > 0 else 'none_injured'
logger.info(key='Percent_of_shock_in_rti',
data={'Percent_of_shock_in_rti': percent_in_shock},
description='The percentage of those assigned injuries who were also assign the shock property')
@@ -3342,13 +3203,8 @@ def apply(self, person_id, squeeze_factor):
self.sim.population.props.at[person_id, 'rt_diagnosed'] = True
road_traffic_injuries = self.sim.modules['RTI']
road_traffic_injuries.rti_injury_diagnosis(person_id, self.EXPECTED_APPT_FOOTPRINT)
-
- if 'DiagRadio' in list(self.EXPECTED_APPT_FOOTPRINT.keys()):
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('X-ray'))
-
- elif 'Tomography' in list(self.EXPECTED_APPT_FOOTPRINT.keys()):
+ if 'Tomography' in list(self.EXPECTED_APPT_FOOTPRINT.keys()):
self.ACCEPTED_FACILITY_LEVEL = '3'
- self.add_equipment({'Computed Tomography (CT machine)', 'CT scanner accessories'})
def did_not_run(self, *args, **kwargs):
pass
@@ -3650,9 +3506,6 @@ def apply(self, person_id, squeeze_factor):
# determine the number of ICU days used to treat patient
if df.loc[person_id, 'rt_ISS_score'] > self.hdu_cut_off_iss_score:
-
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('ICU'))
-
mean_icu_days = p['mean_icu_days']
sd_icu_days = p['sd_icu_days']
mean_tbi_icu_days = p['mean_tbi_icu_days']
@@ -3945,6 +3798,8 @@ class HSI_RTI_Shock_Treatment(HSI_Event, IndividualScopeEventMixin):
"""
This HSI event handles the process of treating hypovolemic shock, as recommended by the pediatric
handbook for Malawi and (TODO: FIND ADULT REFERENCE)
+ Currently this HSI_Event is described only and not used, as I still need to work out how to model the occurrence
+ of shock
"""
def __init__(self, module, person_id):
@@ -3954,12 +3809,9 @@ def __init__(self, module, person_id):
self.TREATMENT_ID = 'Rti_ShockTreatment'
self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({'AccidentsandEmerg': 1})
self.ACCEPTED_FACILITY_LEVEL = '1b'
- self._number_of_times_this_event_has_run = 0
- self._maximum_number_times_event_should_run = self.module.parameters['maximum_number_of_times_HSI_events_should_run']
def apply(self, person_id, squeeze_factor):
df = self.sim.population.props
- self._number_of_times_this_event_has_run += 1
# determine if this is a child
if df.loc[person_id, 'age_years'] < 15:
is_child = True
@@ -3967,25 +3819,36 @@ def apply(self, person_id, squeeze_factor):
is_child = False
if not df.at[person_id, 'is_alive']:
return self.make_appt_footprint({})
-
+ get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name
# TODO: find a more complete list of required consumables for adults
if is_child:
+ self.module.item_codes_for_consumables_required['shock_treatment_child'] = {
+ get_item_code("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 1,
+ get_item_code("Dextrose (glucose) 5%, 1000ml_each_CMST"): 1,
+ get_item_code('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
+ get_item_code('Blood, one unit'): 1,
+ get_item_code("Oxygen, 1000 liters, primarily with oxygen cylinders"): 1
+ }
is_cons_available = self.get_consumables(
- self.module.cons_item_codes['shock_treatment_child']
+ self.module.item_codes_for_consumables_required['shock_treatment_child']
)
else:
+ self.module.item_codes_for_consumables_required['shock_treatment_adult'] = {
+ get_item_code("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 1,
+ get_item_code('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
+ get_item_code('Blood, one unit'): 1,
+ get_item_code("Oxygen, 1000 liters, primarily with oxygen cylinders"): 1
+ }
is_cons_available = self.get_consumables(
- self.module.cons_item_codes['shock_treatment_adult']
+ self.module.item_codes_for_consumables_required['shock_treatment_adult']
)
if is_cons_available:
logger.debug(key='rti_general_message',
data=f"Hypovolemic shock treatment available for person {person_id}")
df.at[person_id, 'rt_in_shock'] = False
- self.add_equipment({'Infusion pump', 'Drip stand', 'Oxygen cylinder, with regulator', 'Nasal Prongs'})
else:
- if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run:
- self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
+ self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
return self.make_appt_footprint({})
def did_not_run(self):
@@ -4038,21 +3901,17 @@ def __init__(self, module, person_id):
self.TREATMENT_ID = 'Rti_FractureCast'
self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({'AccidentsandEmerg': 1})
self.ACCEPTED_FACILITY_LEVEL = '1b'
- self._number_of_times_this_event_has_run = 0
- self._maximum_number_times_event_should_run = self.module.parameters[
- 'maximum_number_of_times_HSI_events_should_run']
def apply(self, person_id, squeeze_factor):
# Get the population and health system
df = self.sim.population.props
p = df.loc[person_id]
- self._number_of_times_this_event_has_run += 1
-
# if the person isn't alive return a blank footprint
if not df.at[person_id, 'is_alive']:
return self.make_appt_footprint({})
# get a shorthand reference to RTI and consumables modules
road_traffic_injuries = self.sim.modules['RTI']
+ get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name
# isolate the relevant injury information
# Find the untreated injuries
untreated_injury_cols = _get_untreated_injury_columns(person_id, df)
@@ -4073,22 +3932,20 @@ def apply(self, person_id, squeeze_factor):
assert len(p['rt_injuries_to_cast']) > 0
# Check this injury assigned to be treated here is actually had by the person
assert all(injuries in person_injuries.values for injuries in p['rt_injuries_to_cast'])
-
- # If they have a fracture that needs a cast, ask for consumables, updating to match the number of
- # fractures).
+ # If they have a fracture that needs a cast, ask for plaster of paris
+ self.module.item_codes_for_consumables_required['fracture_treatment'] = {
+ get_item_code('Plaster of Paris (POP) 10cm x 7.5cm slab_12_CMST'): fracturecastcounts,
+ get_item_code('Bandage, crepe 7.5cm x 1.4m long , when stretched'): slingcounts,
+ }
is_cons_available = self.get_consumables(
- self.module.cons_item_codes['fracture_treatment'](fracturecastcounts)
+ self.module.item_codes_for_consumables_required['fracture_treatment']
)
-
# if the consumables are available then the appointment can run
if is_cons_available:
logger.debug(key='rti_general_message',
data=f"Fracture casts available for person %d's {fracturecastcounts + slingcounts} fractures, "
f"{person_id}"
)
-
- self.add_equipment({'Casting platform', 'Casting chairs', 'Bucket, 10L'})
-
# update the property rt_med_int to indicate they are recieving treatment
df.at[person_id, 'rt_med_int'] = True
# Find the persons injuries
@@ -4140,8 +3997,7 @@ def apply(self, person_id, squeeze_factor):
df.loc[person_id, 'rt_injuries_to_cast'].clear()
df.loc[person_id, 'rt_date_death_no_med'] = pd.NaT
else:
- if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run:
- self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
+ self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
if pd.isnull(df.loc[person_id, 'rt_date_death_no_med']):
df.loc[person_id, 'rt_date_death_no_med'] = self.sim.date + DateOffset(days=7)
logger.debug(key='rti_general_message',
@@ -4181,16 +4037,13 @@ def __init__(self, module, person_id):
self.TREATMENT_ID = 'Rti_OpenFractureTreatment'
self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({'MinorSurg': 1})
self.ACCEPTED_FACILITY_LEVEL = '1b'
- self._number_of_times_this_event_has_run = 0
- self._maximum_number_times_event_should_run = self.module.parameters[
- 'maximum_number_of_times_HSI_events_should_run']
def apply(self, person_id, squeeze_factor):
df = self.sim.population.props
- self._number_of_times_this_event_has_run += 1
if not df.at[person_id, 'is_alive']:
return self.make_appt_footprint({})
road_traffic_injuries = self.sim.modules['RTI']
+ get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name
# isolate the relevant injury information
person_injuries = df.loc[[person_id], RTI.INJURY_COLUMNS]
# check if they have a fracture that requires a cast
@@ -4203,24 +4056,31 @@ def apply(self, person_id, squeeze_factor):
assert df.loc[person_id, 'rt_med_int'], 'person sent here has not been treated'
# If they have an open fracture, ask for consumables to treat fracture
- wound_contaminated = (
- (open_fracture_counts > 0)
- and (self.module.parameters['prob_open_fracture_contaminated'] > self.module.rng.random_sample())
- )
-
+ if open_fracture_counts > 0:
+ self.module.item_codes_for_consumables_required['open_fracture_treatment'] = {
+ get_item_code('Ceftriaxone 1g, PFR_each_CMST'): 1,
+ get_item_code('Cetrimide 15% + chlorhexidine 1.5% solution.for dilution _5_CMST'): 1,
+ get_item_code("Gauze, absorbent 90cm x 40m_each_CMST"): 1,
+ get_item_code('Suture pack'): 1,
+ }
+ # If wound is "grossly contaminated" administer Metronidazole
+ # todo: parameterise the probability of wound contamination
+ p = self.module.parameters
+ prob_open_fracture_contaminated = p['prob_open_fracture_contaminated']
+ rand_for_contamination = self.module.rng.random_sample(size=1)
+ if rand_for_contamination < prob_open_fracture_contaminated:
+ self.module.item_codes_for_consumables_required['open_fracture_treatment'].update(
+ {get_item_code('Metronidazole, injection, 500 mg in 100 ml vial'): 1}
+ )
# Check that there are enough consumables to treat this person's fractures
- is_cons_available = self.get_consumables(self.module.cons_item_codes["open_fracture_treatment"]) and (
- # If wound is "grossly contaminated" administer Metronidazole, else ignore
- self.get_consumables(self.module.cons_item_codes["open_fracture_treatment_additional_if_contaminated"])
- if wound_contaminated else True)
+ is_cons_available = self.get_consumables(
+ self.module.item_codes_for_consumables_required['open_fracture_treatment']
+ )
if is_cons_available:
logger.debug(key='rti_general_message',
data=f"Fracture casts available for person {person_id} {open_fracture_counts} open fractures"
)
-
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery'))
-
person = df.loc[person_id]
# update the dataframe to show this person is recieving treatment
df.loc[person_id, 'rt_med_int'] = True
@@ -4247,8 +4107,7 @@ def apply(self, person_id, squeeze_factor):
if code[0] in df.loc[person_id, 'rt_injuries_for_open_fracture_treatment']:
df.loc[person_id, 'rt_injuries_for_open_fracture_treatment'].remove(code[0])
else:
- if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run:
- self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
+ self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
if pd.isnull(df.loc[person_id, 'rt_date_death_no_med']):
df.loc[person_id, 'rt_date_death_no_med'] = self.sim.date + DateOffset(days=7)
logger.debug(key='rti_general_message',
@@ -4291,14 +4150,10 @@ def __init__(self, module, person_id):
self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({
('Under5OPD' if self.sim.population.props.at[person_id, "age_years"] < 5 else 'Over5OPD'): 1})
self.ACCEPTED_FACILITY_LEVEL = '1b'
- self._number_of_times_this_event_has_run = 0
- self._maximum_number_times_event_should_run = self.module.parameters[
- 'maximum_number_of_times_HSI_events_should_run']
def apply(self, person_id, squeeze_factor):
+ get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name
df = self.sim.population.props
- self._number_of_times_this_event_has_run += 1
-
if not df.at[person_id, 'is_alive']:
return self.make_appt_footprint({})
road_traffic_injuries = self.sim.modules['RTI']
@@ -4312,10 +4167,15 @@ def apply(self, person_id, squeeze_factor):
# Check that the person sent here has an injury that is treated by this HSI event
assert lacerationcounts > 0
if lacerationcounts > 0:
+ self.module.item_codes_for_consumables_required['laceration_treatment'] = {
+ get_item_code('Suture pack'): lacerationcounts,
+ get_item_code('Cetrimide 15% + chlorhexidine 1.5% solution.for dilution _5_CMST'): lacerationcounts,
+ }
# check the number of suture kits required and request them
is_cons_available = self.get_consumables(
- self.module.cons_item_codes['laceration_treatment'](lacerationcounts))
+ self.module.item_codes_for_consumables_required['laceration_treatment']
+ )
# Availability of consumables determines if the intervention is delivered...
if is_cons_available:
@@ -4338,8 +4198,7 @@ def apply(self, person_id, squeeze_factor):
assert df.loc[person_id, date_to_remove_daly_column] > self.sim.date
df.loc[person_id, 'rt_date_death_no_med'] = pd.NaT
else:
- if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run:
- self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
+ self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
if pd.isnull(df.loc[person_id, 'rt_date_death_no_med']):
df.loc[person_id, 'rt_date_death_no_med'] = self.sim.date + DateOffset(days=7)
logger.debug(key='rti_general_message',
@@ -4386,14 +4245,10 @@ def __init__(self, module, person_id):
p = self.module.parameters
self.prob_mild_burns = p['prob_mild_burns']
- self._number_of_times_this_event_has_run = 0
- self._maximum_number_times_event_should_run = p['maximum_number_of_times_HSI_events_should_run']
-
def apply(self, person_id, squeeze_factor):
+ get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name
df = self.sim.population.props
- self._number_of_times_this_event_has_run += 1
-
if not df.at[person_id, 'is_alive']:
return self.make_appt_footprint({})
road_traffic_injuries = self.sim.modules['RTI']
@@ -4408,8 +4263,11 @@ def apply(self, person_id, squeeze_factor):
assert df.loc[person_id, 'rt_med_int'], 'this person has not been treated'
if burncounts > 0:
# Request materials for burn treatment
- cons_needed = self.module.cons_item_codes['burn_treatment'](burncounts)
+ self.module.item_codes_for_consumables_required['burn_treatment'] = {
+ get_item_code("Gauze, absorbent 90cm x 40m_each_CMST"): burncounts,
+ get_item_code('Cetrimide 15% + chlorhexidine 1.5% solution.for dilution _5_CMST'): burncounts,
+ }
possible_large_TBSA_burn_codes = ['7113', '8113', '4113', '5113']
idx2, bigburncounts = \
road_traffic_injuries.rti_find_and_count_injuries(person_injuries, possible_large_TBSA_burn_codes)
@@ -4418,11 +4276,13 @@ def apply(self, person_id, squeeze_factor):
if (burncounts > 1) or ((len(idx2) > 0) & (random_for_severe_burn > self.prob_mild_burns)):
# check if they have multiple burns, which implies a higher burned total body surface area (TBSA) which
# will alter the treatment plan
- cons_needed.update(
- self.module.cons_item_codes['ringers lactate for multiple burns']
+ self.module.item_codes_for_consumables_required['burn_treatment'].update(
+ {get_item_code("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 1}
)
- is_cons_available = self.get_consumables(cons_needed)
+ is_cons_available = self.get_consumables(
+ self.module.item_codes_for_consumables_required['burn_treatment']
+ )
if is_cons_available:
logger.debug(key='rti_general_message',
data=f"This facility has burn treatment available which has been used for person "
@@ -4461,8 +4321,7 @@ def apply(self, person_id, squeeze_factor):
)
df.loc[person_id, 'rt_date_death_no_med'] = pd.NaT
else:
- if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run:
- self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
+ self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
if pd.isnull(df.loc[person_id, 'rt_date_death_no_med']):
df.loc[person_id, 'rt_date_death_no_med'] = self.sim.date + DateOffset(days=7)
logger.debug(key='rti_general_message',
@@ -4489,14 +4348,9 @@ def __init__(self, module, person_id):
self.TREATMENT_ID = 'Rti_TetanusVaccine'
self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({'EPI': 1})
self.ACCEPTED_FACILITY_LEVEL = '1b'
- self._number_of_times_this_event_has_run = 0
- self._maximum_number_times_event_should_run = self.module.parameters[
- 'maximum_number_of_times_HSI_events_should_run']
def apply(self, person_id, squeeze_factor):
df = self.sim.population.props
- self._number_of_times_this_event_has_run += 1
-
if not df.at[person_id, 'is_alive']:
return self.make_appt_footprint({})
person_injuries = df.loc[[person_id], RTI.INJURY_COLUMNS]
@@ -4514,13 +4368,18 @@ def apply(self, person_id, squeeze_factor):
return self.make_appt_footprint({})
# If they have a laceration/burn ask request the tetanus vaccine
if counts > 0:
- is_tetanus_available = self.get_consumables(self.module.cons_item_codes['tetanus_treatment'])
+ get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name
+ self.module.item_codes_for_consumables_required['tetanus_treatment'] = {
+ get_item_code('Tetanus toxoid, injection'): 1
+ }
+ is_tetanus_available = self.get_consumables(
+ self.module.item_codes_for_consumables_required['tetanus_treatment']
+ )
if is_tetanus_available:
logger.debug(key='rti_general_message',
data=f"Tetanus vaccine requested for person {person_id} and given")
else:
- if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run:
- self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
+ self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
logger.debug(key='rti_general_message',
data=f"Tetanus vaccine requested for person {person_id}, not given")
return self.make_appt_footprint({})
@@ -4550,20 +4409,16 @@ def __init__(self, module, person_id):
self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({
('Under5OPD' if self.sim.population.props.at[person_id, "age_years"] < 5 else 'Over5OPD'): 1})
self.ACCEPTED_FACILITY_LEVEL = '1b'
- self._number_of_times_this_event_has_run = 0
- self._maximum_number_times_event_should_run = self.module.parameters[
- 'maximum_number_of_times_HSI_events_should_run']
def apply(self, person_id, squeeze_factor):
df = self.sim.population.props
- self._number_of_times_this_event_has_run += 1
-
if not df.at[person_id, 'is_alive']:
return self.make_appt_footprint({})
# Check that the person sent here is alive, has been through A&E and RTI_Med_int
assert df.loc[person_id, 'rt_diagnosed'], 'This person has not been through a and e'
assert df.loc[person_id, 'rt_med_int'], 'This person has not been through rti med int'
person_injuries = df.loc[[person_id], RTI.INJURY_COLUMNS]
+ get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name
road_traffic_injuries = self.sim.modules['RTI']
pain_level = "none"
# create a dictionary to associate the level of pain to the codes
@@ -4607,12 +4462,25 @@ def apply(self, person_id, squeeze_factor):
data=dict_to_output,
description='Summary of the pain medicine requested by each person')
if df.loc[person_id, 'age_years'] < 16:
+ self.module.item_codes_for_consumables_required['pain_management'] = {
+ get_item_code("Paracetamol 500mg_1000_CMST"): 1
+ }
cond = self.get_consumables(
- self.module.cons_item_codes['pain_management_mild_under_16']
+ self.module.item_codes_for_consumables_required['pain_management']
)
else:
- cond1 = self.get_consumables(self.module.cons_item_codes['pain_management_mild_above_16'])
- cond2 = self.get_consumables(self.module.cons_item_codes['pain_management_mild_under_16'])
+ self.module.item_codes_for_consumables_required['pain_management'] = {
+ get_item_code("diclofenac sodium 25 mg, enteric coated_1000_IDA"): 1
+ }
+ cond1 = self.get_consumables(
+ self.module.item_codes_for_consumables_required['pain_management']
+ )
+ self.module.item_codes_for_consumables_required['pain_management'] = {
+ get_item_code("Paracetamol 500mg_1000_CMST"): 1
+ }
+ cond2 = self.get_consumables(
+ self.module.item_codes_for_consumables_required['pain_management']
+ )
if (cond1 is True) & (cond2 is True):
which = self.module.rng.random_sample(size=1)
if which <= 0.5:
@@ -4652,8 +4520,7 @@ def apply(self, person_id, squeeze_factor):
data=dict_to_output,
description='Pain medicine successfully provided to the person')
else:
- if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run:
- self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
+ self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
logger.debug(key='rti_general_message',
data=f"This facility has no pain management available for their mild pain, person "
f"{person_id}.")
@@ -4665,8 +4532,12 @@ def apply(self, person_id, squeeze_factor):
logger.info(key='Requested_Pain_Management',
data=dict_to_output,
description='Summary of the pain medicine requested by each person')
-
- is_cons_available = self.get_consumables(self.module.cons_item_codes['pain_management_moderate'])
+ self.module.item_codes_for_consumables_required['pain_management'] = {
+ get_item_code("tramadol HCl 100 mg/2 ml, for injection_100_IDA"): 1
+ }
+ is_cons_available = self.get_consumables(
+ self.module.item_codes_for_consumables_required['pain_management']
+ )
logger.debug(key='rti_general_message',
data=f"Person {person_id} has requested tramadol for moderate pain relief")
@@ -4680,8 +4551,7 @@ def apply(self, person_id, squeeze_factor):
data=dict_to_output,
description='Pain medicine successfully provided to the person')
else:
- if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run:
- self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
+ self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
logger.debug(key='rti_general_message',
data=f"This facility has no pain management available for moderate pain for person "
f"{person_id}.")
@@ -4694,8 +4564,11 @@ def apply(self, person_id, squeeze_factor):
data=dict_to_output,
description='Summary of the pain medicine requested by each person')
# give morphine
+ self.module.item_codes_for_consumables_required['pain_management'] = {
+ get_item_code("morphine sulphate 10 mg/ml, 1 ml, injection (nt)_10_IDA"): 1
+ }
is_cons_available = self.get_consumables(
- self.module.cons_item_codes['pain_management_severe']
+ self.module.item_codes_for_consumables_required['pain_management']
)
logger.debug(key='rti_general_message',
data=f"Person {person_id} has requested morphine for severe pain relief")
@@ -4710,8 +4583,7 @@ def apply(self, person_id, squeeze_factor):
data=dict_to_output,
description='Pain medicine successfully provided to the person')
else:
- if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run:
- self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
+ self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
logger.debug(key='rti_general_message',
data=f"This facility has no pain management available for severe pain for person "
f"{person_id}.")
@@ -4839,8 +4711,6 @@ def __init__(self, module, person_id):
self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({'MajorSurg': 1})
self.ACCEPTED_FACILITY_LEVEL = '1b'
self.BEDDAYS_FOOTPRINT = self.make_beddays_footprint({})
- self._number_of_times_this_event_has_run = 0
- self._maximum_number_times_event_should_run = self.module.parameters['maximum_number_of_times_HSI_events_should_run']
p = self.module.parameters
self.prob_perm_disability_with_treatment_severe_TBI = p['prob_perm_disability_with_treatment_severe_TBI']
@@ -4848,14 +4718,38 @@ def __init__(self, module, person_id):
self.treated_code = 'none'
def apply(self, person_id, squeeze_factor):
- self._number_of_times_this_event_has_run += 1
df = self.sim.population.props
rng = self.module.rng
road_traffic_injuries = self.sim.modules['RTI']
-
+ get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name
# Request first draft of consumables used in major surgery
+ self.module.item_codes_for_consumables_required['major_surgery'] = {
+ # request a general anaesthetic
+ get_item_code("Halothane (fluothane)_250ml_CMST"): 1,
+ # clean the site of the surgery
+ get_item_code("Chlorhexidine 1.5% solution_5_CMST"): 1,
+ # tools to begin surgery
+ get_item_code("Scalpel blade size 22 (individually wrapped)_100_CMST"): 1,
+ # administer an IV
+ get_item_code('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
+ get_item_code("Giving set iv administration + needle 15 drops/ml_each_CMST"): 1,
+ get_item_code("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 1,
+ # repair incision made
+ get_item_code("Suture pack"): 1,
+ get_item_code("Gauze, absorbent 90cm x 40m_each_CMST"): 1,
+ # administer pain killer
+ get_item_code('Pethidine, 50 mg/ml, 2 ml ampoule'): 1,
+ # administer antibiotic
+ get_item_code("Ampicillin injection 500mg, PFR_each_CMST"): 1,
+ # equipment used by surgeon, gloves and facemask
+ get_item_code('Disposables gloves, powder free, 100 pieces per box'): 1,
+ get_item_code('surgical face mask, disp., with metal nose piece_50_IDA'): 1,
+ # request syringe
+ get_item_code("Syringe, Autodisable SoloShot IX "): 1
+ }
+
request_outcome = self.get_consumables(
- self.module.cons_item_codes['major_surgery']
+ self.module.item_codes_for_consumables_required['major_surgery']
)
if not df.at[person_id, 'is_alive']:
@@ -4905,9 +4799,6 @@ def apply(self, person_id, squeeze_factor):
# RTI_Med
assert df.loc[person_id, 'rt_diagnosed'], 'This person has not been through a and e'
assert df.loc[person_id, 'rt_med_int'], 'This person has not been through rti med int'
-
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery'))
-
# ------------------------ Track permanent disabilities with treatment -------------------------------------
# --------------------------------- Perm disability from TBI -----------------------------------------------
codes = ['133', '133a', '133b', '133c', '133d', '134', '134a', '134b', '135']
@@ -5096,8 +4987,7 @@ def apply(self, person_id, squeeze_factor):
['Treated injury code not removed', self.treated_code]
df.loc[person_id, 'rt_date_death_no_med'] = pd.NaT
else:
- if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run:
- self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
+ self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
if pd.isnull(df.loc[person_id, 'rt_date_death_no_med']):
df.loc[person_id, 'rt_date_death_no_med'] = self.sim.date + DateOffset(days=7)
return self.make_appt_footprint({})
@@ -5163,16 +5053,36 @@ def __init__(self, module, person_id):
self.EXPECTED_APPT_FOOTPRINT = self.make_appt_footprint({'MinorSurg': 1})
self.ACCEPTED_FACILITY_LEVEL = '1b'
- self._number_of_times_this_event_has_run = 0
- self._maximum_number_times_event_should_run = self.module.parameters[
- 'maximum_number_of_times_HSI_events_should_run']
-
def apply(self, person_id, squeeze_factor):
- self._number_of_times_this_event_has_run += 1
df = self.sim.population.props
if not df.at[person_id, 'is_alive']:
return self.make_appt_footprint({})
-
+ get_item_code = self.sim.modules['HealthSystem'].get_item_code_from_item_name
+ # Request first draft of consumables used in major surgery
+ self.module.item_codes_for_consumables_required['minor_surgery'] = {
+ # request a local anaesthetic
+ get_item_code("Halothane (fluothane)_250ml_CMST"): 1,
+ # clean the site of the surgery
+ get_item_code("Chlorhexidine 1.5% solution_5_CMST"): 1,
+ # tools to begin surgery
+ get_item_code("Scalpel blade size 22 (individually wrapped)_100_CMST"): 1,
+ # administer an IV
+ get_item_code('Cannula iv (winged with injection pot) 18_each_CMST'): 1,
+ get_item_code("Giving set iv administration + needle 15 drops/ml_each_CMST"): 1,
+ get_item_code("ringer's lactate (Hartmann's solution), 1000 ml_12_IDA"): 1,
+ # repair incision made
+ get_item_code("Suture pack"): 1,
+ get_item_code("Gauze, absorbent 90cm x 40m_each_CMST"): 1,
+ # administer pain killer
+ get_item_code('Pethidine, 50 mg/ml, 2 ml ampoule'): 1,
+ # administer antibiotic
+ get_item_code("Ampicillin injection 500mg, PFR_each_CMST"): 1,
+ # equipment used by surgeon, gloves and facemask
+ get_item_code('Disposables gloves, powder free, 100 pieces per box'): 1,
+ get_item_code('surgical face mask, disp., with metal nose piece_50_IDA'): 1,
+ # request syringe
+ get_item_code("Syringe, Autodisable SoloShot IX "): 1
+ }
rng = self.module.rng
road_traffic_injuries = self.sim.modules['RTI']
surgically_treated_codes = ['322', '211', '212', '323', '722', '291', '241', '811', '812', '813a', '813b',
@@ -5198,12 +5108,12 @@ def apply(self, person_id, squeeze_factor):
treated_code = rng.choice(relevant_codes)
# need to determine whether this person has an injury which will treated with external fixation
# external_fixation_codes = ['811', '812', '813a', '813b', '813c']
- request_outcome = self.get_consumables(self.module.cons_item_codes['minor_surgery'])
+ request_outcome = self.get_consumables(
+ self.module.item_codes_for_consumables_required['minor_surgery']
+ )
# todo: think about consequences of certain consumables not being available for minor surgery and model health
# outcomes
if request_outcome:
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('Major Surgery'))
-
# create a dictionary to store the recovery times for each injury in days
minor_surg_recov_time_days = {
'322': 180,
@@ -5262,8 +5172,7 @@ def apply(self, person_id, squeeze_factor):
['Injury treated not removed', treated_code]
df.loc[person_id, 'rt_date_death_no_med'] = pd.NaT
else:
- if self._number_of_times_this_event_has_run < self._maximum_number_times_event_should_run:
- self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
+ self.sim.modules['RTI'].schedule_hsi_event_for_tomorrow(self)
if pd.isnull(df.loc[person_id, 'rt_date_death_no_med']):
df.loc[person_id, 'rt_date_death_no_med'] = self.sim.date + DateOffset(days=7)
logger.debug(key='rti_general_message',
@@ -5580,7 +5489,7 @@ def apply(self, population):
label: (
len(pop_subset.loc[pop_subset['rt_inj_severity'] == 'severe'])
/ len(pop_subset)
- ) if len(pop_subset) > 0 else float("nan")
+ ) if len(pop_subset) > 0 else "none_injured"
for label, pop_subset in population_subsets_with_injuries.items()
}
self.totmild += (population_with_injuries.rt_inj_severity == "mild").sum()
@@ -5596,25 +5505,25 @@ def apply(self, population):
description='severity of injuries in simulation')
# ==================================== Incidence ==============================================================
# How many were involved in a RTI
- n_in_RTI = int(df.rt_road_traffic_inc.sum())
+ n_in_RTI = df.rt_road_traffic_inc.sum()
children_in_RTI = len(df.loc[df.rt_road_traffic_inc & (df['age_years'] < 19)])
children_alive = len(df.loc[df['age_years'] < 19])
self.numerator += n_in_RTI
self.totinjured += n_in_RTI
# How many were disabled
- n_perm_disabled = int((df.is_alive & df.rt_perm_disability).sum())
+ n_perm_disabled = (df.is_alive & df.rt_perm_disability).sum()
# self.permdis += n_perm_disabled
- n_alive = int(df.is_alive.sum())
+ n_alive = df.is_alive.sum()
self.denominator += (n_alive - n_in_RTI) * (1 / 12)
- n_immediate_death = int((df.rt_road_traffic_inc & df.rt_imm_death).sum())
+ n_immediate_death = (df.rt_road_traffic_inc & df.rt_imm_death).sum()
self.deathonscene += n_immediate_death
diedfromrtiidx = df.index[df.rt_imm_death | df.rt_post_med_death | df.rt_no_med_death | df.rt_death_from_shock |
df.rt_unavailable_med_death]
- n_sought_care = int((df.rt_road_traffic_inc & df.rt_med_int).sum())
+ n_sought_care = (df.rt_road_traffic_inc & df.rt_med_int).sum()
self.soughtmedcare += n_sought_care
- n_death_post_med = int(df.rt_post_med_death.sum())
+ n_death_post_med = df.rt_post_med_death.sum()
self.deathaftermed += n_death_post_med
- self.deathwithoutmed += int(df.rt_no_med_death.sum())
+ self.deathwithoutmed += df.rt_no_med_death.sum()
self.death_inc_numerator += n_immediate_death + n_death_post_med + len(df.loc[df.rt_no_med_death])
self.death_in_denominator += (n_alive - (n_immediate_death + n_death_post_med + len(df.loc[df.rt_no_med_death])
)) * \
@@ -5623,7 +5532,7 @@ def apply(self, population):
percent_accidents_result_in_death = \
(self.deathonscene + self.deathaftermed + self.deathwithoutmed) / self.numerator
else:
- percent_accidents_result_in_death = float("nan")
+ percent_accidents_result_in_death = 'none injured'
maleinrti = len(df.loc[df.rt_road_traffic_inc & (df['sex'] == 'M')])
femaleinrti = len(df.loc[df.rt_road_traffic_inc & (df['sex'] == 'F')])
@@ -5632,35 +5541,35 @@ def apply(self, population):
maleinrti = maleinrti / divider
femaleinrti = femaleinrti / divider
else:
- maleinrti = 1.0
- femaleinrti = 0.0
+ maleinrti = 1
+ femaleinrti = 0
mfratio = [maleinrti, femaleinrti]
if (n_in_RTI - len(df.loc[df.rt_imm_death])) > 0:
percent_sought_care = n_sought_care / (n_in_RTI - len(df.loc[df.rt_imm_death]))
else:
- percent_sought_care = float("nan")
+ percent_sought_care = 'none_injured'
if n_sought_care > 0:
percent_died_post_care = n_death_post_med / n_sought_care
else:
- percent_died_post_care = float("nan")
+ percent_died_post_care = 'none_injured'
if n_sought_care > 0:
percentage_admitted_to_ICU_or_HDU = len(df.loc[df.rt_med_int & df.rt_in_icu_or_hdu]) / n_sought_care
else:
- percentage_admitted_to_ICU_or_HDU = float("nan")
+ percentage_admitted_to_ICU_or_HDU = 'none_injured'
if (n_alive - n_in_RTI) > 0:
inc_rti = (n_in_RTI / ((n_alive - n_in_RTI) * (1 / 12))) * 100000
else:
- inc_rti = 0.0
+ inc_rti = 0
if (children_alive - children_in_RTI) > 0:
inc_rti_in_children = (children_in_RTI / ((children_alive - children_in_RTI) * (1 / 12))) * 100000
else:
- inc_rti_in_children = 0.0
+ inc_rti_in_children = 0
if (n_alive - len(diedfromrtiidx)) > 0:
inc_rti_death = (len(diedfromrtiidx) / ((n_alive - len(diedfromrtiidx)) * (1 / 12))) * 100000
else:
- inc_rti_death = 0.0
+ inc_rti_death = 0
if (n_alive - len(df.loc[df.rt_post_med_death])) > 0:
inc_post_med_death = (len(df.loc[df.rt_post_med_death]) / ((n_alive - len(df.loc[df.rt_post_med_death])) *
(1 / 12))) * 100000
@@ -5670,21 +5579,21 @@ def apply(self, population):
inc_imm_death = (len(df.loc[df.rt_imm_death]) / ((n_alive - len(df.loc[df.rt_imm_death])) * (1 / 12))) * \
100000
else:
- inc_imm_death = 0.0
+ inc_imm_death = 0
if (n_alive - len(df.loc[df.rt_no_med_death])) > 0:
inc_death_no_med = (len(df.loc[df.rt_no_med_death]) /
((n_alive - len(df.loc[df.rt_no_med_death])) * (1 / 12))) * 100000
else:
- inc_death_no_med = 0.0
+ inc_death_no_med = 0
if (n_alive - len(df.loc[df.rt_unavailable_med_death])) > 0:
inc_death_unavailable_med = (len(df.loc[df.rt_unavailable_med_death]) /
((n_alive - len(df.loc[df.rt_unavailable_med_death])) * (1 / 12))) * 100000
else:
- inc_death_unavailable_med = 0.0
+ inc_death_unavailable_med = 0
if self.fracdenominator > 0:
frac_incidence = (self.totfracnumber / self.fracdenominator) * 100000
else:
- frac_incidence = 0.0
+ frac_incidence = 0
# calculate case fatality ratio for those injured who don't seek healthcare
did_not_seek_healthcare = len(df.loc[df.rt_road_traffic_inc & ~df.rt_med_int & ~df.rt_diagnosed])
died_no_healthcare = \
@@ -5692,12 +5601,12 @@ def apply(self, population):
if did_not_seek_healthcare > 0:
cfr_no_med = died_no_healthcare / did_not_seek_healthcare
else:
- cfr_no_med = float("nan")
+ cfr_no_med = 'all_sought_care'
# calculate incidence rate per 100,000 of deaths on scene
if n_alive > 0:
inc_death_on_scene = (len(df.loc[df.rt_imm_death]) / n_alive) * 100000 * (1 / 12)
else:
- inc_death_on_scene = 0.0
+ inc_death_on_scene = 0
dict_to_output = {
'number involved in a rti': n_in_RTI,
'incidence of rti per 100,000': inc_rti,
@@ -5735,7 +5644,7 @@ def apply(self, population):
percent_related_to_alcohol = len(injuredDemographics.loc[injuredDemographics.li_ex_alc]) / \
len(injuredDemographics)
except ZeroDivisionError:
- percent_related_to_alcohol = 0.0
+ percent_related_to_alcohol = 0
injured_demography_summary = {
'males_in_rti': injuredDemographics['sex'].value_counts()['M'],
'females_in_rti': injuredDemographics['sex'].value_counts()['F'],
diff --git a/src/tlo/methods/schisto.py b/src/tlo/methods/schisto.py
index 0e9735286a..8ccc593601 100644
--- a/src/tlo/methods/schisto.py
+++ b/src/tlo/methods/schisto.py
@@ -1,24 +1,19 @@
-from __future__ import annotations
-
from pathlib import Path
-from typing import TYPE_CHECKING, List, Optional, Sequence, Union
+from typing import List, Optional, Sequence, Union
import numpy as np
import pandas as pd
from tlo import Date, DateOffset, Module, Parameter, Property, Types, logging
from tlo.analysis.utils import flatten_multi_index_series_into_dict_for_logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.methods import Metadata
from tlo.methods.causes import Cause
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
from tlo.methods.symptommanager import Symptom
from tlo.util import random_date
-if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
-
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
@@ -31,7 +26,7 @@
_AGE_GROUPS = {'PSAC': (0, 4), 'SAC': (5, 14), 'Adults': (15, 120), 'All': (0, 120)}
-class Schisto(Module, GenericFirstAppointmentsMixin):
+class Schisto(Module):
"""Schistosomiasis module.
Two species of worm that cause Schistosomiasis are modelled independently. Worms are acquired by persons via the
environment. There is a delay between the acquisition of worms and the maturation to 'adults' worms; and a long
@@ -331,20 +326,19 @@ def _schedule_mda_events(self) -> None:
def do_at_generic_first_appt(
self,
- person_id: int,
+ patient_id: int,
symptoms: List[str],
- schedule_hsi_event: HSIEventScheduler,
**kwargs
- ) -> None:
+ ) -> IndividualPropertyUpdates:
# Do when person presents to the GenericFirstAppt.
# If the person has certain set of symptoms, refer ta HSI for testing.
set_of_symptoms_indicative_of_schisto = {'anemia', 'haematuria', 'bladder_pathology'}
if set_of_symptoms_indicative_of_schisto.issubset(symptoms):
event = HSI_Schisto_TestingFollowingSymptoms(
- module=self, person_id=person_id
+ module=self, person_id=patient_id
)
- schedule_hsi_event(event, priority=0, topen=self.sim.date)
+ self.healthsystem.schedule_hsi_event(event, priority=0, topen=self.sim.date)
class SchistoSpecies:
@@ -923,8 +917,6 @@ def apply(self, person_id, squeeze_factor):
)
if will_test:
- self.add_equipment({'Ordinary Microscope'})
-
# Determine if they truly are infected (with any of the species)
is_infected = (person.loc[cols_of_infection_status] != 'Non-infected').any()
diff --git a/src/tlo/methods/stunting.py b/src/tlo/methods/stunting.py
index ec2725bd39..bacb9f4d7d 100644
--- a/src/tlo/methods/stunting.py
+++ b/src/tlo/methods/stunting.py
@@ -19,15 +19,14 @@
from scipy.stats import norm
from tlo import DAYS_IN_YEAR, DateOffset, Module, Parameter, Property, Types, logging
+from tlo.core import IndividualPropertyUpdates
from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata
from tlo.methods.hsi_event import HSI_Event
-from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
if TYPE_CHECKING:
- from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
- from tlo.population import IndividualProperties
+ from tlo.population import PatientDetails
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
@@ -37,7 +36,7 @@
# MODULE DEFINITION
# ---------------------------------------------------------------------------------------------------------
-class Stunting(Module, GenericFirstAppointmentsMixin):
+class Stunting(Module):
"""This is the disease module for Stunting"""
INIT_DEPENDENCIES = {'Demography', 'Wasting', 'NewbornOutcomes', 'Diarrhoea', 'Hiv'}
@@ -288,31 +287,30 @@ def do_treatment(self, person_id, prob_success):
def do_at_generic_first_appt(
self,
- person_id: int,
- individual_properties: IndividualProperties,
- schedule_hsi_event: HSIEventScheduler,
+ patient_id: int,
+ patient_details: PatientDetails,
**kwargs,
- ) -> None:
- # This is called by the a generic HSI event for every child aged less than 5
- # years. It assesses stunting and schedules an HSI as needed.
- is_stunted = individual_properties["un_HAZ_category"] in (
- "HAZ<-3",
- "-3<=HAZ<-2",
- )
- p_stunting_diagnosed = self.parameters[
- "prob_stunting_diagnosed_at_generic_appt"
- ]
-
- # Schedule the HSI for provision of treatment based on the probability of
- # stunting diagnosis, provided the necessary symptoms are there.
- if individual_properties["age_years"] <= 5 and is_stunted:
- # Schedule the HSI for provision of treatment based on the probability of
- # stunting diagnosis
+ ) -> IndividualPropertyUpdates:
+ # This is called by the a generic HSI event for every child aged
+ # less than 5 years.
+ # It assesses stunting and schedules an HSI as needed.
+ is_stunted = patient_details.un_HAZ_category in ('HAZ<-3', '-3<=HAZ<-2')
+ p_stunting_diagnosed = self.parameters['prob_stunting_diagnosed_at_generic_appt']
+
+ # Schedule the HSI for provision of treatment based on the
+ # probability of stunting diagnosis, provided the necessary
+ # symptoms are there.
+ if (
+ (patient_details.age_years <= 5)
+ and is_stunted
+ ):
+ # Schedule the HSI for provision of treatment based on the
+ # probability of stunting diagnosis
if p_stunting_diagnosed > self.rng.random_sample():
event = HSI_Stunting_ComplementaryFeeding(
- module=self, person_id=person_id
+ module=self, person_id=patient_id
)
- schedule_hsi_event(
+ self.healthsystem.schedule_hsi_event(
event,
priority=2, # <-- lower priority that for wasting and most other HSI
topen=self.sim.date,
@@ -524,9 +522,7 @@ def apply(self, population):
"""Log the current distribution of stunting classification by age"""
df = population.props
- subset = df.loc[df.is_alive & (df.age_years < 5)].copy()
- subset["age_years"] = pd.Categorical(subset["age_years"], categories=range(5))
- d_to_log = subset.groupby(
+ d_to_log = df.loc[df.is_alive & (df.age_years < 5)].groupby(
by=['age_years', 'un_HAZ_category']).size().sort_index().to_dict()
def convert_keys_to_string(d):
diff --git a/src/tlo/methods/symptommanager.py b/src/tlo/methods/symptommanager.py
index 67389e283e..68edbf0840 100644
--- a/src/tlo/methods/symptommanager.py
+++ b/src/tlo/methods/symptommanager.py
@@ -11,11 +11,9 @@
* The probability of spurious symptoms is not informed by data.
"""
-from __future__ import annotations
-
from collections import defaultdict
from pathlib import Path
-from typing import TYPE_CHECKING, List, Optional, Sequence, Union
+from typing import Sequence, Union
import numpy as np
import pandas as pd
@@ -25,9 +23,6 @@
from tlo.methods import Metadata
from tlo.util import BitsetHandler
-if TYPE_CHECKING:
- from tlo.population import IndividualProperties
-
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
@@ -277,8 +272,7 @@ def pre_initialise_population(self):
SymptomManager.PROPERTIES = dict()
for symptom_name in sorted(self.symptom_names):
symptom_column_name = self.get_column_name_for_symptom(symptom_name)
- SymptomManager.PROPERTIES[symptom_column_name] = Property(Types.BITSET,
- f'Presence of symptom {symptom_name}')
+ SymptomManager.PROPERTIES[symptom_column_name] = Property(Types.BITSET, f'Presence of symptom {symptom_name}')
def initialise_population(self, population):
"""
@@ -465,81 +459,33 @@ def who_not_have(self, symptom_string: str) -> pd.Index:
)
]
- def has_what(
- self,
- person_id: Optional[int] = None,
- individual_details: Optional[IndividualProperties] = None,
- disease_module: Optional[Module] = None,
- ) -> List[str]:
+ def has_what(self, person_id, disease_module: Module = None):
"""
This is a helper function that will give a list of strings for the symptoms that a _single_ person
is currently experiencing.
+ Optionally can specify disease_module_name to limit to the symptoms caused by that disease module
- If working in a `tlo.population.IndividualProperties` context, one can pass the context object
- instead of supplying the person's DataFrame index.
- Note that at least one of these inputs must be passed as a keyword argument however.
- In the event that both arguments are passed, the individual_details argument takes precedence over the person_id.
-
- Optionally can specify disease_module_name to limit to the symptoms caused by that disease module.
-
- :param person_id: the person_of of interest.
- :param individual_details: `tlo.population.IndividualProperties` object for the person of interest.
- :param disease_module: (optional) disease module of interest.
- :return: list of strings for the symptoms that are currently being experienced.
+ :param person_id: the person_of of interest
+ :param disease_module: (optional) disease module of interest
+ :return: list of strings for the symptoms that are currently being experienced
"""
- assert (
- disease_module.name in ([self.name] + self.recognised_module_names)
- if disease_module is not None
- else True
- ), "Disease Module Name is not recognised"
-
- if individual_details is not None:
- # We are working in an IndividualDetails context, avoid lookups to the
- # population DataFrame as we have this context stored already.
- assert individual_details["is_alive"], "The person is not alive"
-
- if disease_module is not None:
- int_repr = self.bsh._element_to_int_map[disease_module.name]
- return [
- symptom
- for symptom in self.symptom_names
- if individual_details[
- self.bsh._get_columns(self.get_column_name_for_symptom(symptom))
- ]
- & int_repr
- != 0
- ]
- else:
- return [
- symptom
- for symptom in self.symptom_names
- if individual_details[self.get_column_name_for_symptom(symptom)] > 0
- ]
- else:
- assert isinstance(
- person_id, (int, np.integer)
- ), "person_id must be a single integer for one particular person"
- df = self.sim.population.props
- assert df.at[person_id, "is_alive"], "The person is not alive"
+ assert isinstance(person_id, (int, np.integer)), 'person_id must be a single integer for one particular person'
- if disease_module is not None:
- sy_columns = [
- self.get_column_name_for_symptom(s) for s in self.symptom_names
- ]
- person_has = self.bsh.has(
- [person_id], disease_module.name, first=True, columns=sy_columns
- )
- return [s for s in self.symptom_names if person_has[f"sy_{s}"]]
- else:
- symptom_cols = df.loc[
- person_id, [f"sy_{s}" for s in self.symptom_names]
- ]
- return (
- symptom_cols.index[symptom_cols > 0]
- .str.removeprefix("sy_")
- .to_list()
- )
+ df = self.sim.population.props
+ assert df.at[person_id, 'is_alive'], "The person is not alive"
+
+ if disease_module is not None:
+ assert disease_module.name in ([self.name] + self.recognised_module_names), \
+ "Disease Module Name is not recognised"
+ sy_columns = [self.get_column_name_for_symptom(s) for s in self.symptom_names]
+ person_has = self.bsh.has(
+ [person_id], disease_module.name, first=True, columns=sy_columns
+ )
+ return [s for s in self.symptom_names if person_has[f'sy_{s}']]
+ else:
+ symptom_cols = df.loc[person_id, [f'sy_{s}' for s in self.symptom_names]]
+ return symptom_cols.index[symptom_cols > 0].str.removeprefix("sy_").to_list()
def have_what(self, person_ids: Sequence[int]):
"""Find the set of symptoms for a list of person_ids.
diff --git a/src/tlo/methods/tb.py b/src/tlo/methods/tb.py
index 623ee2e483..b769e60fb9 100644
--- a/src/tlo/methods/tb.py
+++ b/src/tlo/methods/tb.py
@@ -9,7 +9,7 @@
import pandas as pd
-from tlo import Date, DateOffset, Module, Parameter, Property, Types, logging
+from tlo import DateOffset, Module, Parameter, Property, Types, logging
from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
from tlo.lm import LinearModel, LinearModelType, Predictor
from tlo.methods import Metadata, hiv
@@ -376,19 +376,6 @@ def __init__(self, name=None, resourcefilepath=None, run_with_checks=False):
Types.LIST,
"length of inpatient stay for end-of-life TB patients",
),
- # ------------------ scale-up parameters for scenario analysis ------------------ #
- "type_of_scaleup": Parameter(
- Types.STRING, "argument to determine type scale-up of program which will be implemented, "
- "can be 'none', 'target' or 'max'",
- ),
- "scaleup_start_year": Parameter(
- Types.INT,
- "the year when the scale-up starts (it will occur on 1st January of that year)"
- ),
- "scaleup_parameters": Parameter(
- Types.DATA_FRAME,
- "the parameters and values changed in scenario analysis"
- )
}
def read_parameters(self, data_folder):
@@ -426,9 +413,6 @@ def read_parameters(self, data_folder):
.tolist()
)
- # load parameters for scale-up projections
- p['scaleup_parameters'] = workbook["scaleup_parameters"]
-
# 2) Get the DALY weights
if "HealthBurden" in self.sim.modules.keys():
# HIV-negative
@@ -470,13 +454,9 @@ def read_parameters(self, data_folder):
)
def pre_initialise_population(self):
- """Do things required before the population is created
- * Build the LinearModels"""
- self._build_linear_models()
-
- def _build_linear_models(self):
- """Establish the Linear Models"""
-
+ """
+ * Establish the Linear Models
+ """
p = self.parameters
# risk of active tb
@@ -763,7 +743,6 @@ def get_consumables_for_dx_and_tx(self):
)
# 4) -------- Define the treatment options --------
- # treatment supplied as full kits for duration of treatment
# adult treatment - primary
self.item_codes_for_consumables_required['tb_tx_adult'] = \
hs.get_item_code_from_item_name("Cat. I & III Patient Kit A")
@@ -781,16 +760,12 @@ def get_consumables_for_dx_and_tx(self):
hs.get_item_code_from_item_name("Cat. II Patient Kit A2")
# mdr treatment
- self.item_codes_for_consumables_required['tb_mdrtx'] = \
- hs.get_item_code_from_item_name("Treatment: second-line drugs")
+ self.item_codes_for_consumables_required['tb_mdrtx'] = {
+ hs.get_item_code_from_item_name("Treatment: second-line drugs"): 1}
# ipt
- self.item_codes_for_consumables_required['tb_ipt'] = \
- hs.get_item_code_from_item_name("Isoniazid/Pyridoxine, tablet 300 mg")
-
- # 3hp
- self.item_codes_for_consumables_required['tb_3HP'] = \
- hs.get_item_code_from_item_name("Isoniazid/Rifapentine")
+ self.item_codes_for_consumables_required['tb_ipt'] = {
+ hs.get_item_code_from_item_name("Isoniazid/Pyridoxine, tablet 300 mg"): 1}
def initialise_population(self, population):
@@ -869,13 +844,6 @@ def initialise_simulation(self, sim):
sim.schedule_event(TbSelfCureEvent(self), sim.date)
sim.schedule_event(TbActiveCasePoll(self), sim.date + DateOffset(years=1))
- # 2) log at the end of the year
- # Optional: Schedule the scale-up of programs
- if self.parameters["type_of_scaleup"] != 'none':
- scaleup_start_date = Date(self.parameters["scaleup_start_year"], 1, 1)
- assert scaleup_start_date >= self.sim.start_date, f"Date {scaleup_start_date} is before simulation starts."
- sim.schedule_event(TbScaleUpEvent(self), scaleup_start_date)
-
# 2) log at the end of the year
sim.schedule_event(TbLoggingEvent(self), sim.date + DateOffset(years=1))
@@ -888,37 +856,6 @@ def initialise_simulation(self, sim):
TbCheckPropertiesEvent(self), sim.date + pd.DateOffset(months=1)
)
- def update_parameters_for_program_scaleup(self):
- """ options for program scale-up are 'target' or 'max' """
- p = self.parameters
- scaled_params_workbook = p["scaleup_parameters"]
-
- if p['type_of_scaleup'] == 'target':
- scaled_params = scaled_params_workbook.set_index('parameter')['target_value'].to_dict()
- else:
- scaled_params = scaled_params_workbook.set_index('parameter')['max_value'].to_dict()
-
- # scale-up TB program
- # use NTP treatment rates
- p["rate_testing_active_tb"]["treatment_coverage"] = scaled_params["tb_treatment_coverage"]
-
- # increase tb treatment success rates
- p["prob_tx_success_ds"] = scaled_params["tb_prob_tx_success_ds"]
- p["prob_tx_success_mdr"] = scaled_params["tb_prob_tx_success_mdr"]
- p["prob_tx_success_0_4"] = scaled_params["tb_prob_tx_success_0_4"]
- p["prob_tx_success_5_14"] = scaled_params["tb_prob_tx_success_5_14"]
-
- # change first-line testing for TB to xpert
- p["first_line_test"] = scaled_params["first_line_test"]
- p["second_line_test"] = scaled_params["second_line_test"]
-
- # increase coverage of IPT
- p["ipt_coverage"]["coverage_plhiv"] = scaled_params["ipt_coverage_plhiv"]
- p["ipt_coverage"]["coverage_paediatric"] = scaled_params["ipt_coverage_paediatric"]
-
- # update exising linear models to use new scaled-up paramters
- self._build_linear_models()
-
def on_birth(self, mother_id, child_id):
"""Initialise properties for a newborn individual
allocate IPT for child if mother diagnosed with TB
@@ -1425,21 +1362,6 @@ def apply(self, population):
self.module.relapse_event(population)
-class TbScaleUpEvent(Event, PopulationScopeEventMixin):
- """ This event exists to change parameters or functions
- depending on the scenario for projections which has been set
- It only occurs once on date: scaleup_start_date,
- called by initialise_simulation
- """
-
- def __init__(self, module):
- super().__init__(module)
-
- def apply(self, population):
-
- self.module.update_parameters_for_program_scaleup()
-
-
class TbActiveEvent(RegularEvent, PopulationScopeEventMixin):
"""
* check for those with dates of active tb onset within last time-period
@@ -1540,7 +1462,7 @@ def apply(self, population):
)
# -------- 5) schedule screening for asymptomatic and symptomatic people --------
- # sample from all NEW active cases (active_idx) and determine whether they will seek a test
+ # sample from all new active cases (active_idx) and determine whether they will seek a test
year = min(2019, max(2011, now.year))
active_testing_rates = p["rate_testing_active_tb"]
@@ -1683,19 +1605,8 @@ def apply(self, person_id, squeeze_factor):
p = self.module.parameters
person = df.loc[person_id]
- if not person["is_alive"]:
- return self.sim.modules["HealthSystem"].get_blank_appt_footprint()
-
- # If the person is already diagnosed, do nothing do not occupy any resources
- if person["tb_diagnosed"]:
- return self.sim.modules["HealthSystem"].get_blank_appt_footprint()
-
- # If the person is already on treatment and not failing, do nothing do not occupy any resources
- if person["tb_on_treatment"] and not person["tb_treatment_failure"]:
- return self.sim.modules["HealthSystem"].get_blank_appt_footprint()
-
- # if person has tested within last 14 days, do nothing
- if person["tb_date_tested"] >= (self.sim.date - DateOffset(days=7)):
+ # If the person is dead or already diagnosed, do nothing do not occupy any resources
+ if not person["is_alive"] or person["tb_diagnosed"]:
return self.sim.modules["HealthSystem"].get_blank_appt_footprint()
logger.debug(
@@ -1704,11 +1615,15 @@ def apply(self, person_id, squeeze_factor):
smear_status = person["tb_smear"]
+ # If the person is already on treatment and not failing, do nothing do not occupy any resources
+ if person["tb_on_treatment"] and not person["tb_treatment_failure"]:
+ return self.sim.modules["HealthSystem"].get_blank_appt_footprint()
+
# ------------------------- screening ------------------------- #
# check if patient has: cough, fever, night sweat, weight loss
# if none of the above conditions are present, no further action
- persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id=person_id)
+ persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id)
if not any(x in self.module.symptom_list for x in persons_symptoms):
return self.make_appt_footprint({})
@@ -1786,9 +1701,6 @@ def apply(self, person_id, squeeze_factor):
].dx_manager.run_dx_test(
dx_tests_to_run="tb_clinical", hsi_event=self
)
- if test_result is not None:
- # Add used equipment
- self.add_equipment({'Sputum Collection box', 'Ordinary Microscope'})
elif test == "xpert":
@@ -1821,9 +1733,6 @@ def apply(self, person_id, squeeze_factor):
dx_tests_to_run="tb_xpert_test_smear_negative",
hsi_event=self,
)
- if test_result is not None:
- # Add used equipment
- self.add_equipment({'Sputum Collection box', 'Gene Expert (16 Module)'})
# ------------------------- testing referrals ------------------------- #
@@ -1842,9 +1751,6 @@ def apply(self, person_id, squeeze_factor):
ACTUAL_APPT_FOOTPRINT = self.make_appt_footprint(
{"Over5OPD": 2, "LabTBMicro": 1}
)
- if test_result is not None:
- # Add used equipment
- self.add_equipment({'Sputum Collection box', 'Ordinary Microscope'})
# if still no result available, rely on clinical diagnosis
if test_result is None:
@@ -1971,7 +1877,7 @@ def apply(self, person_id, squeeze_factor):
# check if patient has: cough, fever, night sweat, weight loss
set_of_symptoms_that_indicate_tb = set(self.module.symptom_list)
- persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id=person_id)
+ persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id)
if not set_of_symptoms_that_indicate_tb.intersection(persons_symptoms):
# if none of the above conditions are present, no further action
@@ -2042,8 +1948,6 @@ def apply(self, person_id, squeeze_factor):
test_result = self.sim.modules["HealthSystem"].dx_manager.run_dx_test(
dx_tests_to_run="tb_xray_smear_negative", hsi_event=self
)
- if test_result is not None:
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('X-ray'))
# if consumables not available, refer to level 2
# return blank footprint as xray did not occur
@@ -2117,8 +2021,6 @@ def apply(self, person_id, squeeze_factor):
test_result = self.sim.modules["HealthSystem"].dx_manager.run_dx_test(
dx_tests_to_run="tb_xray_smear_negative", hsi_event=self
)
- if test_result is not None:
- self.add_equipment(self.healthcare_system.equipment.from_pkg_names('X-ray'))
# if consumables not available, rely on clinical diagnosis
# return blank footprint as xray was not available
@@ -2200,9 +2102,8 @@ def apply(self, person_id, squeeze_factor):
return self.sim.modules["HealthSystem"].get_blank_appt_footprint()
treatment_regimen = self.select_treatment(person_id)
- # treatment supplied in kits, one kit per treatment course
treatment_available = self.get_consumables(
- item_codes={self.module.item_codes_for_consumables_required[treatment_regimen]: 1}
+ item_codes=self.module.item_codes_for_consumables_required[treatment_regimen]
)
# if require MDR treatment, and not currently at level 2, refer to level 2
@@ -2380,9 +2281,6 @@ def apply(self, person_id, squeeze_factor):
test_result = self.sim.modules["HealthSystem"].dx_manager.run_dx_test(
dx_tests_to_run="tb_sputum_test_smear_negative", hsi_event=self
)
- if test_result is not None:
- # Add used equipment
- self.add_equipment({'Sputum Collection box', 'Ordinary Microscope'})
# if sputum test was available and returned positive and not diagnosed with mdr, schedule xpert test
if test_result and not person["tb_diagnosed_mdr"]:
@@ -2397,9 +2295,6 @@ def apply(self, person_id, squeeze_factor):
xperttest_result = self.sim.modules["HealthSystem"].dx_manager.run_dx_test(
dx_tests_to_run="tb_xpert_test_smear_negative", hsi_event=self
)
- if xperttest_result is not None:
- # Add used equipment
- self.add_equipment({'Sputum Collection box', 'Gene Expert (16 Module)'})
# if xpert test returns new mdr-tb diagnosis
if xperttest_result and (df.at[person_id, "tb_strain"] == "mdr"):
@@ -2445,9 +2340,8 @@ class HSI_Tb_Start_or_Continue_Ipt(HSI_Event, IndividualScopeEventMixin):
* HIV.HSI_Hiv_StartOrContinueTreatment for PLHIV, diagnosed and on ART
* Tb.HSI_Tb_StartTreatment for up to 5 contacts of diagnosed active TB case
- Isoniazid preventive therapy for HIV-infected children : 6 months, 180 doses
- 3HP (Isoniazid/Rifapentine) for adults: 12 weeks, 12 doses
- 3HP for children ages >2 yrs hiv-
+ if person referred by ART initiation (HIV+), IPT given for 36 months
+ paediatric IPT is 6-9 months
"""
def __init__(self, module, person_id):
@@ -2475,7 +2369,7 @@ def apply(self, person_id, squeeze_factor):
return
# if currently have symptoms of TB, refer for screening/testing
- persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id=person_id)
+ persons_symptoms = self.sim.modules["SymptomManager"].has_what(person_id)
if any(x in self.module.symptom_list for x in persons_symptoms):
self.sim.modules["HealthSystem"].schedule_hsi_event(
@@ -2487,23 +2381,10 @@ def apply(self, person_id, squeeze_factor):
else:
# Check/log use of consumables, and give IPT if available
-
- # if child and HIV+ or child under 2 yrs
- if ((person["age_years"] <= 15) and person["hv_inf"]) or (person["age_years"] <= 2):
-
- # 6 months dispensation, once daily
- drugs_available = self.get_consumables(
- item_codes={self.module.item_codes_for_consumables_required["tb_ipt"]: 180})
-
- # for all others
- else:
- # 12 weeks dispensation, once weekly
- drugs_available = self.get_consumables(
- item_codes={self.module.item_codes_for_consumables_required["tb_3HP"]: 12}
- )
-
- # if available, schedule IPT decision
- if drugs_available:
+ # if not available, reschedule IPT start
+ if self.get_consumables(
+ item_codes=self.module.item_codes_for_consumables_required["tb_ipt"]
+ ):
# Update properties
df.at[person_id, "tb_on_ipt"] = True
df.at[person_id, "tb_date_ipt"] = self.sim.date
@@ -2729,7 +2610,7 @@ def apply(self, population):
)
# proportion of active TB cases in the last year who are HIV-positive
- prop_hiv = inc_active_hiv / new_tb_cases if new_tb_cases else 0.0
+ prop_hiv = inc_active_hiv / new_tb_cases if new_tb_cases else 0
logger.info(
key="tb_incidence",
@@ -2763,7 +2644,7 @@ def apply(self, population):
df[(df.age_years >= 15) & df.is_alive]
) if len(
df[(df.age_years >= 15) & df.is_alive]
- ) else 0.0
+ ) else 0
assert prev_active_adult <= 1
# prevalence of active TB in children
@@ -2774,7 +2655,7 @@ def apply(self, population):
df[(df.age_years < 15) & df.is_alive]
) if len(
df[(df.age_years < 15) & df.is_alive]
- ) else 0.0
+ ) else 0
assert prev_active_child <= 1
# LATENT
@@ -2791,7 +2672,7 @@ def apply(self, population):
df[(df.age_years >= 15) & df.is_alive]
) if len(
df[(df.age_years >= 15) & df.is_alive]
- ) else 0.0
+ ) else 0
assert prev_latent_adult <= 1
# proportion of population with latent TB - children
@@ -2833,7 +2714,7 @@ def apply(self, population):
if new_mdr_cases:
prop_mdr = new_mdr_cases / new_tb_cases
else:
- prop_mdr = 0.0
+ prop_mdr = 0
logger.info(
key="tb_mdr",
@@ -2855,7 +2736,7 @@ def apply(self, population):
if new_tb_diagnosis:
prop_dx = new_tb_diagnosis / new_tb_cases
else:
- prop_dx = 0.0
+ prop_dx = 0
# ------------------------------------ TREATMENT ------------------------------------
# number of tb cases who became active in last timeperiod and initiated treatment
@@ -2871,7 +2752,7 @@ def apply(self, population):
tx_coverage = new_tb_tx / new_tb_cases
# assert tx_coverage <= 1
else:
- tx_coverage = 0.0
+ tx_coverage = 0
# ipt coverage
new_tb_ipt = len(
@@ -2884,7 +2765,7 @@ def apply(self, population):
if new_tb_ipt:
current_ipt_coverage = new_tb_ipt / len(df[df.is_alive])
else:
- current_ipt_coverage = 0.0
+ current_ipt_coverage = 0
logger.info(
key="tb_treatment",
@@ -2955,7 +2836,7 @@ def apply(self, population):
if adult_num_false_positive:
adult_prop_false_positive = adult_num_false_positive / new_tb_tx_adult
else:
- adult_prop_false_positive = 0.0
+ adult_prop_false_positive = 0
# children
child_num_false_positive = len(
diff --git a/src/tlo/population.py b/src/tlo/population.py
index 37f5fccfdf..74ae0692e7 100644
--- a/src/tlo/population.py
+++ b/src/tlo/population.py
@@ -1,81 +1,39 @@
-"""Types for representing a properties of a population of individuals."""
+"""The Person and Population classes."""
import math
-from collections.abc import Generator
-from contextlib import contextmanager
-from typing import Any, Dict, Optional, Set
+from typing import Any, Dict
import pandas as pd
-from tlo import Property, logging
+from tlo import logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class IndividualProperties:
- """Memoized view of population dataframe row that is optionally read-only.
-
- This class should not be instantiated directly but instead the
- :py:meth:`Population.individual_properties` context manager method used to create
- instances for a given population.
- """
-
- def __init__(
- self, population_dataframe: pd.DataFrame, person_id: int, read_only: bool = True
- ):
- self._finalized = False
- self._read_only = read_only
+class PatientDetails:
+ """Read-only memoized view of population dataframe row."""
+
+ def __init__(self, population_dataframe: pd.DataFrame, person_id: int):
+ self._population_dataframe = population_dataframe
+ self._person_id = person_id
self._property_cache: Dict[str, Any] = {}
- # Avoid storing a reference to population_dataframe internally by mediating
- # access via closures to guard against direct access
- self._get_value_at = lambda key: population_dataframe.at[person_id, key]
- if not read_only:
- self._properties_updated: Set[str] = set()
-
- def synchronize_updates_to_dataframe():
- row_index = population_dataframe.index.get_loc(person_id)
- for key in self._properties_updated:
- # This chained indexing approach to setting dataframe values is
- # significantly (~3 to 4 times) quicker than using at / iat
- # indexers, but will fail when copy-on-write is enabled which will
- # be default in Pandas 3
- column = population_dataframe[key]
- column.values[row_index] = self._property_cache[key]
-
- self._synchronize_updates_to_dataframe = synchronize_updates_to_dataframe
-
+
def __getitem__(self, key: str) -> Any:
- if self._finalized:
- msg = f"Cannot read value for {key} as instance has been finalized"
- raise ValueError(msg)
try:
return self._property_cache[key]
except KeyError:
- value = self._get_value_at(key)
+ value = self._population_dataframe.at[self._person_id, key]
self._property_cache[key] = value
- return value
-
- def __setitem__(self, key: str, value: Any) -> None:
- if self._finalized:
- msg = f"Cannot set value for {key} as instance has been finalized"
- raise ValueError(msg)
- if self._read_only:
- msg = f"Cannot set value for {key} as destination is read-only"
- raise ValueError(msg)
- self._properties_updated.add(key)
- self._property_cache[key] = value
-
- def synchronize_updates_to_dataframe(self) -> None:
- """Synchronize values for any updated properties to population dataframe."""
- if not self._read_only:
- self._synchronize_updates_to_dataframe()
- self._properties_updated.clear()
-
- def finalize(self) -> None:
- """Synchronize updates to population dataframe and prevent further access."""
- self.synchronize_updates_to_dataframe()
- self._finalized = True
+ return value
+
+ def __getattr__(self, name: str) -> Any:
+ try:
+ return self[name]
+ except KeyError as e:
+ msg = f"'{type(self).__name__}' object has no attribute '{name}'"
+ raise AttributeError(msg) from e
+
class Population:
@@ -83,41 +41,39 @@ class Population:
Useful properties of a population:
+ `sim`
+ The Simulation instance controlling this population.
+
`props`
A Pandas DataFrame with the properties of all individuals as columns.
"""
__slots__ = (
+ "_patient_details_readonly_type",
"props",
+ "sim",
"initial_size",
"new_row",
"next_person_id",
"new_rows",
)
- def __init__(
- self,
- properties: Dict[str, Property],
- initial_size: int,
- append_size: Optional[int] = None,
- ):
+ def __init__(self, sim, initial_size: int, append_size: int = None):
"""Create a new population.
- This will create the required the population dataframe and initialise
- individual's properties as dataframe columns with 'empty' values. The simulation
- will then call disease modules to fill in suitable starting values.
+ This will create the required the population dataframe and initialise individual's
+ properties as dataframe columns with 'empty' values. The simulation will then call disease
+ modules to fill in suitable starting values.
- :param properties: Dictionary defining properties (columns) to initialise
- population dataframe with, keyed by property name and with values
- :py:class:`Property` instances defining the property type.
- :param initial_size: The initial population size.
- :param append_size: How many rows to append when growing the population
- dataframe (optional).
+ :param sim: the Simulation containing this population
+ :param initial_size: the initial population size
+ :param append_size: how many rows to append when growing the population dataframe (optional)
"""
+ self.sim = sim
self.initial_size = initial_size
# Create empty property arrays
- self.props = self._create_props(initial_size, properties)
+ self.props = self._create_props(initial_size)
if append_size is None:
# approximation based on runs to increase capacity of dataframe ~twice a year
@@ -137,7 +93,7 @@ def __init__(
# use the person_id of the next person to be added to the dataframe to increase capacity
self.next_person_id = initial_size
- def _create_props(self, size: int, properties: Dict[str, Property]) -> pd.DataFrame:
+ def _create_props(self, size):
"""Internal helper function to create a properties dataframe.
:param size: the number of rows to create
@@ -145,7 +101,8 @@ def _create_props(self, size: int, properties: Dict[str, Property]) -> pd.DataFr
return pd.DataFrame(
data={
property_name: property.create_series(property_name, size)
- for property_name, property in properties.items()
+ for module in self.sim.modules.values()
+ for property_name, property in module.PROPERTIES.items()
},
index=pd.RangeIndex(stop=size, name="person"),
)
@@ -197,33 +154,15 @@ def make_test_property(self, name, type_):
size = self.initial_size if self.props.empty else len(self.props)
self.props[name] = prop.create_series(name, size)
- @contextmanager
- def individual_properties(
- self, person_id: int, read_only: bool = True
- ) -> Generator[IndividualProperties, None, None]:
+ def row_in_readonly_form(self, patient_index: int) -> PatientDetails:
"""
- Context manager for a memoized view of a row of the population dataframe.
-
- The view returned represents the properties of an individual with properties
- accessible by indexing using string column names, and lazily read-on demand
- from the population dataframe.
-
- Optionally the view returned may allow updating properties as well as reading.
- In this case on exit from the ``with`` block in which the context is entered,
- any updates to the individual properties will be written back to the population
- dataframe.
+ Extract a lazily evaluated, read-only view of a row of the population dataframe.
- Once the ``with`` block in which the context is entered has been exited the view
- returned will raise an error on any subsequent attempts at reading or writing
- properties.
-
- :param person_id: Row index of the dataframe row to extract.
- :param read_only: Whether view is read-only or allows updating properties. If
- ``True`` :py:meth:`IndividualProperties.synchronize_updates_to_dataframe`
- method needs to be called for any updates to be written back to population
- dataframe.
- :returns: Object allowing memoized access to an individual's properties.
+ The object returned represents the properties of an individual with properties
+ accessible either using dot based attribute access or squared bracket based
+ indexing using string column names.
+
+ :param patient_index: Row index of the dataframe row to extract.
+ :returns: Object allowing read-only access to an individuals properties.
"""
- properties = IndividualProperties(self.props, person_id, read_only=read_only)
- yield properties
- properties.finalize()
+ return PatientDetails(self.props, patient_index)
diff --git a/src/tlo/simulation.py b/src/tlo/simulation.py
index 547edf1d23..544ae0d68f 100644
--- a/src/tlo/simulation.py
+++ b/src/tlo/simulation.py
@@ -1,102 +1,58 @@
"""The main simulation controller."""
-from __future__ import annotations
-
import datetime
import heapq
import itertools
import time
from collections import OrderedDict
from pathlib import Path
-from typing import TYPE_CHECKING, Optional
+from typing import Dict, Optional, Union
import numpy as np
-try:
- import dill
-
- DILL_AVAILABLE = True
-except ImportError:
- DILL_AVAILABLE = False
-
from tlo import Date, Population, logging
-from tlo.dependencies import (
- check_dependencies_present,
- initialise_missing_dependencies,
- topologically_sort_modules,
-)
+from tlo.dependencies import check_dependencies_present, topologically_sort_modules
from tlo.events import Event, IndividualScopeEventMixin
from tlo.progressbar import ProgressBar
-if TYPE_CHECKING:
- from tlo.core import Module
- from tlo.logging.core import LogLevel
-
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
-class SimulationPreviouslyInitialisedError(Exception):
- """Exception raised when trying to initialise an already initialised simulation."""
-
-
-class SimulationNotInitialisedError(Exception):
- """Exception raised when trying to run simulation before initialising."""
-
-
class Simulation:
"""The main control centre for a simulation.
- This class contains the core simulation logic and event queue, and holds references
- to all the information required to run a complete simulation: the population,
- disease modules, etc.
+ This class contains the core simulation logic and event queue, and holds
+ references to all the information required to run a complete simulation:
+ the population, disease modules, etc.
Key attributes include:
- :ivar date: The current simulation date.
- :ivar modules: A dictionary of the disease modules used in this simulation, keyed
- by the module name.
- :ivar population: The population being simulated.
- :ivar rng: The simulation-level random number generator.
-
- .. note::
- Individual modules also have their own random number generator with independent
- state.
+ `date`
+ The current simulation date.
+
+ `modules`
+ A list of the disease modules contributing to this simulation.
+
+ `population`
+ The Population being simulated.
+
+ `rng`
+ The simulation-level random number generator.
+ Note that individual modules also have their own random number generator
+ with independent state.
"""
- def __init__(
- self,
- *,
- start_date: Date,
- seed: Optional[int] = None,
- log_config: Optional[dict] = None,
- show_progress_bar: bool = False,
- resourcefilepath: Optional[Path] = None,
- ):
+ def __init__(self, *, start_date: Date, seed: int = None, log_config: dict = None,
+ show_progress_bar=False):
"""Create a new simulation.
- :param start_date: The date the simulation begins; must be given as
- a keyword parameter for clarity.
- :param seed: The seed for random number generator. class will create one if not
- supplied
- :param log_config: Dictionary specifying logging configuration for this
- simulation. Can have entries: `filename` - prefix for log file name, final
- file name will have a date time appended, if not present default is to not
- output log to a file; `directory` - path to output directory to write log
- file to, default if not specified is to output to the `outputs` folder;
- `custom_levels` - dictionary to set logging levels, '*' can be used as a key
- for all registered modules; `suppress_stdout` - if `True`, suppresses
- logging to standard output stream (default is `False`).
- :param show_progress_bar: Whether to show a progress bar instead of the logger
- output during the simulation.
- :param resourcefilepath: Path to resource files folder. Assign ``None` if no
- path is provided.
-
- .. note::
- The `custom_levels` entry in `log_config` argument can be used to disable
- logging on all disease modules by setting a high level to `*`, and then
- enabling logging on one module of interest by setting a low level, for
- example ``{'*': logging.CRITICAL 'tlo.methods.hiv': logging.INFO}``.
+ :param start_date: the date the simulation begins; must be given as
+ a keyword parameter for clarity
+ :param seed: the seed for random number generator. class will create one if not supplied
+ :param log_config: sets up the logging configuration for this simulation
+ :param show_progress_bar: whether to show a progress bar instead of the logger
+ output during the simulation
"""
# simulation
self.date = self.start_date = start_date
@@ -107,60 +63,44 @@ def __init__(
self.population: Optional[Population] = None
self.show_progress_bar = show_progress_bar
- self.resourcefilepath = resourcefilepath
# logging
if log_config is None:
log_config = {}
self._custom_log_levels = None
- self._log_filepath = self._configure_logging(**log_config)
-
+ self._log_filepath = None
+ self._configure_logging(**log_config)
# random number generator
- seed_from = "auto" if seed is None else "user"
+ seed_from = 'auto' if seed is None else 'user'
self._seed = seed
self._seed_seq = np.random.SeedSequence(seed)
logger.info(
- key="info",
- data=f"Simulation RNG {seed_from} entropy = {self._seed_seq.entropy}",
+ key='info',
+ data=f'Simulation RNG {seed_from} entropy = {self._seed_seq.entropy}'
)
self.rng = np.random.RandomState(np.random.MT19937(self._seed_seq))
- # Whether simulation has been initialised
- self._initialised = False
-
- def _configure_logging(
- self,
- filename: Optional[str] = None,
- directory: Path | str = "./outputs",
- custom_levels: Optional[dict[str, LogLevel]] = None,
- suppress_stdout: bool = False
- ):
- """Configure logging of simulation outputs.
-
- Can write log output to a file in addition the default of `stdout`. Mnimum
- custom levels for each logger can be specified for filtering out messages.
-
- :param filename: Prefix for log file name, final file name will have a date time
- appended.
+ def _configure_logging(self, filename: str = None, directory: Union[Path, str] = "./outputs",
+ custom_levels: Dict[str, int] = None, suppress_stdout: bool = False):
+ """Configure logging, can write logging to a logfile in addition the default of stdout.
+
+ Minimum custom levels for each logger can be specified for filtering out messages
+
+ :param filename: Prefix for logfile name, final logfile will have a datetime appended
:param directory: Path to output directory, default value is the outputs folder.
- :param custom_levels: Dictionary to set logging levels, '*' can be used as a key
- for all registered modules. This is likely to be used to disable logging on
- all disease modules by setting a high level, and then enable one of interest
- by setting a low level, for example
- ``{'*': logging.CRITICAL 'tlo.methods.hiv': logging.INFO}``.
- :param suppress_stdout: If `True`, suppresses logging to standard output stream
- (default is `False`).
+ :param custom_levels: dictionary to set logging levels, '*' can be used as a key for all registered modules.
+ This is likely to be used to disable all disease modules, and then enable one of interest
+ e.g. ``{'*': logging.CRITICAL 'tlo.methods.hiv': logging.INFO}``
+ :param suppress_stdout: If True, suppresses logging to standard output stream (default is False)
:return: Path of the log file if a filename has been given.
"""
# clear logging environment
# if using progress bar we do not print log messages to stdout to avoid
# clashes between progress bar and log output
- logging.initialise(
- add_stdout_handler=not (self.show_progress_bar or suppress_stdout),
- simulation_date_getter=lambda: self.date.isoformat(),
- )
+ logging.init_logging(add_stdout_handler=not (self.show_progress_bar or suppress_stdout))
+ logging.set_simulation(self)
if custom_levels:
# if modules have already been registered
@@ -171,54 +111,39 @@ def _configure_logging(
self._custom_log_levels = custom_levels
if filename and directory:
- timestamp = datetime.datetime.now().strftime("%Y-%m-%dT%H%M%S")
+ timestamp = datetime.datetime.now().strftime('%Y-%m-%dT%H%M%S')
log_path = Path(directory) / f"{filename}__{timestamp}.log"
self.output_file = logging.set_output_file(log_path)
logger.info(key='info', data=f'Log output: {log_path}')
+ self._log_filepath = log_path
return log_path
return None
@property
- def log_filepath(self) -> Path:
+ def log_filepath(self):
"""The path to the log file, if one has been set."""
return self._log_filepath
- def register(
- self,
- *modules: Module,
- sort_modules: bool = True,
- check_all_dependencies: bool = True,
- auto_register_dependencies: bool = False,
- ) -> None:
+ def register(self, *modules, sort_modules=True, check_all_dependencies=True):
"""Register one or more disease modules with the simulation.
- :param modules: The disease module(s) to use as part of this simulation.
+ :param modules: the disease module(s) to use as part of this simulation.
Multiple modules may be given as separate arguments to one call.
:param sort_modules: Whether to topologically sort the modules so that any
initialisation dependencies (specified by the ``INIT_DEPENDENCIES``
attribute) of a module are initialised before the module itself is. A
- :py:exc:`.ModuleDependencyError` exception will be raised if there are
- missing initialisation dependencies or circular initialisation dependencies
- between modules that cannot be resolved. If this flag is set to ``True``
- there is also a requirement that at most one instance of each module is
- registered and :py:exc:`.MultipleModuleInstanceError` will be raised if this
- is not the case.
+ ``ModuleDependencyError`` exception will be raised if there are missing
+ initialisation dependencies or circular initialisation dependencies between
+ modules that cannot be resolved. If this flag is set to ``True`` there is
+ also a requirement that at most one instance of each module is registered
+ and ``MultipleModuleInstanceError`` will be raised if this is not the case.
:param check_all_dependencies: Whether to check if all of each module's declared
dependencies (that is, the union of the ``INIT_DEPENDENCIES`` and
``ADDITIONAL_DEPENDENCIES`` attributes) have been included in the set of
- modules to be registered. A :py:exc:`.ModuleDependencyError` exception will
+ modules to be registered. A ``ModuleDependencyError`` exception will
be raised if there are missing dependencies.
- :param auto_register_dependencies: Whether to register missing module dependencies
- or not. If this argument is set to True, all module dependencies will be
- automatically registered.
"""
- if auto_register_dependencies:
- modules = [
- *modules,
- *initialise_missing_dependencies(modules, resourcefilepath=self.resourcefilepath)
- ]
-
if sort_modules:
modules = list(topologically_sort_modules(modules))
if check_all_dependencies:
@@ -226,32 +151,30 @@ def register(
# Iterate over modules and per-module seed sequences spawned from simulation
# level seed sequence
for module, seed_seq in zip(modules, self._seed_seq.spawn(len(modules))):
- assert (
- module.name not in self.modules
- ), f"A module named {module.name} has already been registered"
+ assert module.name not in self.modules, f'A module named {module.name} has already been registered'
# Seed the RNG for the registered module using spawned seed sequence
logger.info(
- key="info",
+ key='info',
data=(
- f"{module.name} RNG auto (entropy, spawn key) = "
- f"({seed_seq.entropy}, {seed_seq.spawn_key[0]})"
- ),
+ f'{module.name} RNG auto (entropy, spawn key) = '
+ f'({seed_seq.entropy}, {seed_seq.spawn_key[0]})'
+ )
)
module.rng = np.random.RandomState(np.random.MT19937(seed_seq))
self.modules[module.name] = module
module.sim = self
- module.read_parameters("")
+ module.read_parameters('')
if self._custom_log_levels:
logging.set_logging_levels(self._custom_log_levels)
- def make_initial_population(self, *, n: int) -> None:
+ def make_initial_population(self, *, n):
"""Create the initial population to simulate.
- :param n: The number of individuals to create; must be given as
- a keyword parameter for clarity.
+ :param n: the number of individuals to create; must be given as
+ a keyword parameter for clarity
"""
start = time.time()
@@ -260,55 +183,67 @@ def make_initial_population(self, *, n: int) -> None:
module.pre_initialise_population()
# Make the initial population
- properties = {
- name: prop
- for module in self.modules.values()
- for name, prop in module.PROPERTIES.items()
- }
- self.population = Population(properties, n)
+ self.population = Population(self, n)
for module in self.modules.values():
start1 = time.time()
module.initialise_population(self.population)
- logger.debug(
- key="debug",
- data=f"{module.name}.initialise_population() {time.time() - start1} s",
- )
+ logger.debug(key='debug', data=f'{module.name}.initialise_population() {time.time() - start1} s')
end = time.time()
- logger.info(key="info", data=f"make_initial_population() {end - start} s")
+ logger.info(key='info', data=f'make_initial_population() {end - start} s')
- def initialise(self, *, end_date: Date) -> None:
- """Initialise all modules in simulation.
+ def simulate(self, *, end_date):
+ """Simulation until the given end date
- :param end_date: Date to end simulation on - accessible to modules to allow
- initialising data structures which may depend (in size for example) on the
- date range being simulated.
+ :param end_date: when to stop simulating. Only events strictly before this
+ date will be allowed to occur.
+ Must be given as a keyword parameter for clarity.
"""
- if self._initialised:
- msg = "initialise method should only be called once"
- raise SimulationPreviouslyInitialisedError(msg)
- self.date = self.start_date
+ start = time.time()
self.end_date = end_date # store the end_date so that others can reference it
+
for module in self.modules.values():
module.initialise_simulation(self)
- self._initialised = True
- def finalise(self, wall_clock_time: Optional[float] = None) -> None:
- """Finalise all modules in simulation and close logging file if open.
+ progress_bar = None
+ if self.show_progress_bar:
+ num_simulated_days = (end_date - self.start_date).days
+ progress_bar = ProgressBar(
+ num_simulated_days, "Simulation progress", unit="day")
+ progress_bar.start()
+
+ while self.event_queue:
+ event, date = self.event_queue.next_event()
+
+ if self.show_progress_bar:
+ simulation_day = (date - self.start_date).days
+ stats_dict = {
+ "date": str(date.date()),
+ "dataframe size": str(len(self.population.props)),
+ "queued events": str(len(self.event_queue)),
+ }
+ if "HealthSystem" in self.modules:
+ stats_dict["queued HSI events"] = str(
+ len(self.modules["HealthSystem"].HSI_EVENT_QUEUE)
+ )
+ progress_bar.update(simulation_day, stats_dict=stats_dict)
+
+ if date >= end_date:
+ self.date = end_date
+ break
+ self.fire_single_event(event, date)
+
+ # The simulation has ended.
+ if self.show_progress_bar:
+ progress_bar.stop()
- :param wall_clock_time: Optional argument specifying total time taken to
- simulate, to be written out to log before closing.
- """
for module in self.modules.values():
module.on_simulation_end()
- if wall_clock_time is not None:
- logger.info(key="info", data=f"simulate() {wall_clock_time} s")
- self.close_output_file()
- def close_output_file(self) -> None:
- """Close logging file if open."""
+ logger.info(key='info', data=f'simulate() {time.time() - start} s')
+
+ # From Python logging.shutdown
if self.output_file:
- # From Python logging.shutdown
try:
self.output_file.acquire()
self.output_file.flush()
@@ -317,121 +252,52 @@ def close_output_file(self) -> None:
pass
finally:
self.output_file.release()
- self.output_file = None
- def _initialise_progress_bar(self, end_date: Date) -> ProgressBar:
- num_simulated_days = (end_date - self.date).days
- progress_bar = ProgressBar(
- num_simulated_days, "Simulation progress", unit="day"
- )
- progress_bar.start()
- return progress_bar
-
- def _update_progress_bar(self, progress_bar: ProgressBar, date: Date) -> None:
- simulation_day = (date - self.start_date).days
- stats_dict = {
- "date": str(date.date()),
- "dataframe size": str(len(self.population.props)),
- "queued events": str(len(self.event_queue)),
- }
- if "HealthSystem" in self.modules:
- stats_dict["queued HSI events"] = str(
- len(self.modules["HealthSystem"].HSI_EVENT_QUEUE)
- )
- progress_bar.update(simulation_day, stats_dict=stats_dict)
-
- def run_simulation_to(self, *, to_date: Date) -> None:
- """Run simulation up to a specified date.
-
- Unlike :py:meth:`simulate` this method does not initialise or finalise
- simulation and the date simulated to can be any date before or equal to
- simulation end date.
-
- :param to_date: Date to simulate up to but not including - must be before or
- equal to simulation end date specified in call to :py:meth:`initialise`.
- """
- if not self._initialised:
- msg = "Simulation must be initialised before calling run_simulation_to"
- raise SimulationNotInitialisedError(msg)
- if to_date > self.end_date:
- msg = f"to_date {to_date} after simulation end date {self.end_date}"
- raise ValueError(msg)
- if self.show_progress_bar:
- progress_bar = self._initialise_progress_bar(to_date)
- while (
- len(self.event_queue) > 0 and self.event_queue.date_of_next_event < to_date
- ):
- event, date = self.event_queue.pop_next_event_and_date()
- if self.show_progress_bar:
- self._update_progress_bar(progress_bar, date)
- self.fire_single_event(event, date)
- self.date = to_date
- if self.show_progress_bar:
- progress_bar.stop()
-
- def simulate(self, *, end_date: Date) -> None:
- """Simulate until the given end date
-
- :param end_date: When to stop simulating. Only events strictly before this
- date will be allowed to occur. Must be given as a keyword parameter for
- clarity.
- """
- start = time.time()
- self.initialise(end_date=end_date)
- self.run_simulation_to(to_date=end_date)
- self.finalise(time.time() - start)
-
- def schedule_event(self, event: Event, date: Date) -> None:
+ def schedule_event(self, event, date):
"""Schedule an event to happen on the given future date.
- :param event: The event to schedule.
- :param date: wWen the event should happen.
+ :param event: the Event to schedule
+ :param date: when the event should happen
"""
- assert date >= self.date, "Cannot schedule events in the past"
-
- assert "TREATMENT_ID" not in dir(
- event
- ), "This looks like an HSI event. It should be handed to the healthsystem scheduler"
- assert (
- event.__str__().find("HSI_") < 0
- ), "This looks like an HSI event. It should be handed to the healthsystem scheduler"
+ assert date >= self.date, 'Cannot schedule events in the past'
+
+ assert 'TREATMENT_ID' not in dir(event), \
+ 'This looks like an HSI event. It should be handed to the healthsystem scheduler'
+ assert (event.__str__().find('HSI_') < 0), \
+ 'This looks like an HSI event. It should be handed to the healthsystem scheduler'
assert isinstance(event, Event)
self.event_queue.schedule(event=event, date=date)
- def fire_single_event(self, event: Event, date: Date) -> None:
+ def fire_single_event(self, event, date):
"""Fires the event once for the given date
- :param event: :py:class:`Event` to fire.
- :param date: The date of the event.
+ :param event: :py:class:`Event` to fire
+ :param date: the date of the event
"""
self.date = date
event.run()
- def do_birth(self, mother_id: int) -> int:
+ def do_birth(self, mother_id):
"""Create a new child person.
We create a new person in the population and then call the `on_birth` method in
all modules to initialise the child's properties.
- :param mother_id: Row index label of the maternal parent.
- :return: Row index label of the new child.
+ :param mother_id: the maternal parent
+ :return: the new child
"""
child_id = self.population.do_birth()
for module in self.modules.values():
module.on_birth(mother_id, child_id)
return child_id
- def find_events_for_person(self, person_id: int) -> list[tuple[Date, Event]]:
+ def find_events_for_person(self, person_id: int):
"""Find the events in the queue for a particular person.
-
- :param person_id: The row index of the person of interest.
- :return: List of tuples `(date_of_event, event)` for that `person_id` in the
- queue.
-
- .. note::
- This is for debugging and testing only. Not for use in real simulations as it
- is slow.
+ :param person_id: the person_id of interest
+ :returns list of tuples (date_of_event, event) for that person_id in the queue.
+
+ NB. This is for debugging and testing only - not for use in real simulations as it is slow
"""
person_events = []
@@ -442,40 +308,6 @@ def find_events_for_person(self, person_id: int) -> list[tuple[Date, Event]]:
return person_events
- def save_to_pickle(self, pickle_path: Path) -> None:
- """Save simulation state to a pickle file using :py:mod:`dill`.
-
- Requires :py:mod:`dill` to be importable.
-
- :param pickle_path: File path to save simulation state to.
- """
- if not DILL_AVAILABLE:
- raise RuntimeError("Cannot save to pickle as dill is not installed")
- with open(pickle_path, "wb") as pickle_file:
- dill.dump(self, pickle_file)
-
- @staticmethod
- def load_from_pickle(
- pickle_path: Path, log_config: Optional[dict] = None
- ) -> Simulation:
- """Load simulation state from a pickle file using :py:mod:`dill`.
-
- Requires :py:mod:`dill` to be importable.
-
- :param pickle_path: File path to load simulation state from.
- :param log_config: New log configuration to override previous configuration. If
- `None` previous configuration (including output file) will be retained.
-
- :returns: Loaded :py:class:`Simulation` object.
- """
- if not DILL_AVAILABLE:
- raise RuntimeError("Cannot load from pickle as dill is not installed")
- with open(pickle_path, "rb") as pickle_file:
- simulation = dill.load(pickle_file)
- if log_config is not None:
- simulation._log_filepath = simulation._configure_logging(**log_config)
- return simulation
-
class EventQueue:
"""A simple priority queue for events.
@@ -488,32 +320,23 @@ def __init__(self):
self.counter = itertools.count()
self.queue = []
- def schedule(self, event: Event, date: Date) -> None:
+ def schedule(self, event, date):
"""Schedule a new event.
- :param event: The event to schedule.
- :param date: When it should happen.
+ :param event: the event to schedule
+ :param date: when it should happen
"""
entry = (date, event.priority, next(self.counter), event)
heapq.heappush(self.queue, entry)
- def pop_next_event_and_date(self) -> tuple[Event, Date]:
- """Get and remove the earliest event and corresponding date in the queue.
+ def next_event(self):
+ """Get the earliest event in the queue.
- :returns: An `(event, date)` pair.
+ :returns: an (event, date) pair
"""
date, _, _, event = heapq.heappop(self.queue)
return event, date
- @property
- def date_of_next_event(self) -> Date:
- """Get the date of the earliest event in queue without removing from queue.
-
- :returns: Date of next event in queue.
- """
- date, *_ = self.queue[0]
- return date
-
- def __len__(self) -> int:
- """:return: The length of the queue."""
+ def __len__(self):
+ """:return: the length of the queue"""
return len(self.queue)
diff --git a/src/tlo/test/random_birth.py b/src/tlo/test/random_birth.py
index 22a20879b1..950173797d 100644
--- a/src/tlo/test/random_birth.py
+++ b/src/tlo/test/random_birth.py
@@ -68,7 +68,7 @@ def initialise_population(self, population):
# We use 'broadcasting' to set the same value for every individual
df.is_pregnant = False
# We randomly sample birth dates for the initial population during the preceding decade
- start_date = self.sim.date
+ start_date = population.sim.date
dates = pd.date_range(start_date - DateOffset(years=10), start_date, freq='M')
df.date_of_birth = self.rng.choice(dates, size=len(df))
# No children have yet been born. We iterate over the population to ensure each
diff --git a/tests/bitset_handler/__init__.py b/tests/bitset_handler/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/tests/bitset_handler/conftest.py b/tests/bitset_handler/conftest.py
deleted file mode 100644
index 41b6ab3e6f..0000000000
--- a/tests/bitset_handler/conftest.py
+++ /dev/null
@@ -1,95 +0,0 @@
-"""
-Implements the fixtures required in
-https://github.com/pandas-dev/pandas/blob/bdb509f95a8c0ff16530cedb01c2efc822c0d314/pandas/core/dtypes/dtypes.py,
-
-which allows us to run the pandas-provided test suite for custom dtypes.
-Additional tests and fixtures can be defined on top of those listed in the link above, if we want to
-run our own tests.
-"""
-
-from typing import List, Set
-
-import numpy as np
-import pytest
-from numpy.random import PCG64, Generator
-from numpy.typing import NDArray
-
-from tlo.bitset_handler.bitset_extension import BitsetArray, BitsetDtype, ElementType
-
-
-@pytest.fixture(scope="session")
-def _rng() -> Generator:
- return Generator(PCG64(seed=0))
-
-
-@pytest.fixture(scope="session")
-def _set_elements() -> Set[ElementType]:
- return {"1", "2", "3", "4", "5", "a", "b", "c", "d", "e"}
-
-
-@pytest.fixture(scope="session")
-def dtype(_set_elements: Set[ElementType]) -> BitsetDtype:
- return BitsetDtype(_set_elements)
-
-
-@pytest.fixture(scope="session")
-def _1st_3_entries() -> List[Set[ElementType]]:
- """
- We will fix the first 3 entries of the data fixture,
- which is helpful to ensure we have some explicit test
- values that we can directly change if needed.
- """
- return [
- {"1", "e"}, {"a", "d"}, {"2", "4", "5"},
- ]
-
-@pytest.fixture(scope="session")
-def _raw_sets(
- _1st_3_entries: List[Set[ElementType]], _rng: Generator, _set_elements: Set[ElementType]
-) -> List[Set[ElementType]]:
- """
- Length 100 list of sets, the first 3 of which are those in
- the _1st_3_entries fixture. These sets will be used as the
- 'raw_data' for the Bitset Extension test suite.
- """
- set_entries = list(_1st_3_entries)
- elements = list(_set_elements)
- for _ in range(100-len(_1st_3_entries)):
- set_entries.append(
- {
- elements[i]
- for i in _rng.integers(
- 0, len(elements), size=_rng.integers(0, len(elements))
- )
- }
- )
- return set_entries
-
-@pytest.fixture(scope="session")
-def _raw_data(
- _raw_sets: List[Set[ElementType]], dtype: BitsetDtype
-) -> NDArray[np.bytes_]:
- data = np.zeros((100,), dtype=dtype.np_array_dtype)
- for i, set_value in enumerate(_raw_sets):
- data[i] = dtype.as_bytes(set_value)
- return data
-
-
-@pytest.fixture(scope="session")
-def data(
- _raw_data: NDArray[np.bytes_], dtype: BitsetDtype
-) -> BitsetArray:
- return BitsetArray(data=_raw_data, dtype=dtype, copy=True)
-
-
-@pytest.fixture
-def data_for_twos(dtype: BitsetDtype) -> None:
- pytest.skip(f"{dtype} does not support divmod")
-
-
-@pytest.fixture
-def data_missing(dtype: BitsetDtype) -> np.ndarray:
- data = np.zeros((2,), dtype=dtype.np_array_dtype)
- data[0] = dtype.na_value
- data[1] = dtype.as_bytes({"a"})
- return data
diff --git a/tests/bitset_handler/test_bitset_pandas_dtype.py b/tests/bitset_handler/test_bitset_pandas_dtype.py
deleted file mode 100644
index 156f9e49e6..0000000000
--- a/tests/bitset_handler/test_bitset_pandas_dtype.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import re
-
-import pytest
-from pandas.tests.extension.base import BaseDtypeTests
-
-from tlo.bitset_handler.bitset_extension import BitsetDtype
-
-
-class TestBitsetDtype(BaseDtypeTests):
- """
- Setting the dtype fixture, above, to out BitsetDtype results in us inheriting
- all default pandas tests for extension Dtypes.
-
- Additional tests can be added to this class if we so desire.
- """
-
- def test_construct_from_string_another_type_raises(
- self, dtype: BitsetDtype
- ) -> None:
- """
- Reimplementation as the error message we expect is different from that provided
- by base ``pandas`` implementation.
- """
- msg = (
- "Need at least 2 (comma-separated) elements in string to construct bitset."
- )
- with pytest.raises(TypeError, match=re.escape(msg)):
- type(dtype).construct_from_string("another_type")
diff --git a/tests/bitset_handler/test_bitset_set_like_interactions.py b/tests/bitset_handler/test_bitset_set_like_interactions.py
deleted file mode 100644
index 801703ce24..0000000000
--- a/tests/bitset_handler/test_bitset_set_like_interactions.py
+++ /dev/null
@@ -1,162 +0,0 @@
-"""
-Tests for set-like interactions with a pd.Series object of BitsetDtype.
-"""
-import operator
-from typing import Any, Callable, Iterable, List, Set
-
-import pandas as pd
-import pytest
-
-from tlo.bitset_handler.bitset_extension import BitsetDtype, CastableForPandasOps, ElementType
-
-
-def seq_of_sets_to_series(sets: Iterable[Set[ElementType]], dtype: BitsetDtype) -> pd.Series:
- """
- Casts a sequence of sets representing a single BitsetDtype to a
- series with those entries of the appropriate dtype.
- """
- return pd.Series(data=sets, dtype=dtype, copy=True)
-
-
-@pytest.fixture(scope="function")
-def small_series(_1st_3_entries: List[Set[ElementType]], dtype: BitsetDtype):
- """
- Recall that the first 3 entries are always fixed in confest;
- repeating the values here just for ease of reference:
-
- {"1", "e"},
- {"a", "d"},
- {"2", "4", "5"},
- """
- return pd.Series(_1st_3_entries, dtype=dtype, copy=True)
-
-
-@pytest.mark.parametrize(
- ["op", "r_value", "expected"],
- [
- pytest.param(
- [operator.or_, operator.add, operator.sub],
- set(),
- [{"1", "e"}, {"a", "d"}, {"2", "4", "5"}],
- id="ADD, OR, SUB w/ empty set",
- ),
- pytest.param(
- [operator.or_, operator.add],
- "a",
- [{"1", "a", "e"}, {"a", "d"}, {"2", "4", "5", "a"}],
- id="ADD, OR w/ scalar element",
- ),
- pytest.param(
- [operator.or_, operator.add],
- {"1", "2", "a", "d"},
- [
- {"1", "2", "a", "d", "e"},
- {"1", "2", "a", "d"},
- {"1", "2", "4", "5", "a", "d"},
- ],
- id="ADD, OR w/ multiple-entry set",
- ),
- pytest.param(
- operator.and_,
- set(),
- [set()] * 3,
- id="AND w/ empty set",
- ),
- pytest.param(
- operator.and_,
- "a",
- [set(), {"a"}, set()],
- id="AND w/ scalar element",
- ),
- pytest.param(
- operator.and_,
- {"1", "a"},
- [{"1"}, {"a"}, set()],
- id="AND w/ multiple-entry set",
- ),
- pytest.param(
- [operator.eq, operator.le, operator.lt],
- set(),
- pd.Series([False, False, False], dtype=bool),
- id="EQ, LE, LT w/ empty set",
- ),
- pytest.param(
- [operator.eq, operator.le, operator.lt],
- "a",
- pd.Series([False, False, False], dtype=bool),
- id="EQ, LE, LT w/ scalar element",
- ),
- pytest.param(
- [operator.eq, operator.ge, operator.le],
- {"1", "e"},
- pd.Series([True, False, False], dtype=bool),
- id="EQ, GE, LE w/ multiple-entry set",
- ),
- pytest.param(
- [operator.ge, operator.gt],
- set(),
- pd.Series([True, True, True], dtype=bool),
- id="GE, GT w/ empty set",
- ),
- pytest.param(
- [operator.ge, operator.gt],
- "a",
- pd.Series([False, True, False], dtype=bool),
- id="GE, GT w/ scalar element",
- ),
- pytest.param(
- [operator.gt, operator.lt],
- {"1, e"},
- pd.Series([False, False, False], dtype=bool),
- id="GT, LT w/ multiple-entry set",
- ),
- pytest.param(
- operator.sub,
- "a",
- [{"1", "e"}, {"d"}, {"2", "4", "5"}],
- id="SUB w/ scalar element",
- ),
- pytest.param(
- operator.sub,
- {"1", "2", "d", "e"},
- [set(), {"a"}, {"4", "5"}],
- id="SUB w/ multiple-entry set",
- ),
- ],
-)
-def test_series_operation_with_value(
- small_series: pd.Series,
- dtype: BitsetDtype,
- op: List[Callable[[Any, Any], Any]] | Callable[[Any, Any], Any],
- r_value: CastableForPandasOps,
- expected: List[Set[ElementType]] | pd.Series
-) -> None:
- """
- The expected value can be passed in as either a list of sets that will be
- converted to the appropriate pd.Series of bitsets, or as an explicit pd.Series
- of booleans (which is used when testing the comparison operations ==, <=, etc).
-
- If r_value is a scalar, the test will run once using the scalar as the r_value,
- and then again using the cast of the scalar to a set of one element as the r_value.
- - In cases such as this, the two results are expected to be the same,
- which saves us verbiage in the list of test cases above.
- """
- expected = (
- seq_of_sets_to_series(expected, dtype)
- if isinstance(expected, list)
- else expected
- )
-
- if not isinstance(op, list):
- op = [op]
- if isinstance(r_value, ElementType):
- r_values = [r_value, {r_value}]
- else:
- r_values = [r_value]
-
- for operation in op:
- for r_v in r_values:
- result = operation(small_series, r_v)
- assert (
- expected == result
- ).all(), f"Series do not match after operation {operation.__name__} with {r_v} on the right."
diff --git a/tests/conftest.py b/tests/conftest.py
index 33b463343a..47d6c3fa16 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -34,4 +34,4 @@ def pytest_collection_modifyitems(config, items):
def pytest_generate_tests(metafunc):
if "seed" in metafunc.fixturenames:
- metafunc.parametrize("seed", metafunc.config.getoption("seed"), scope="session")
+ metafunc.parametrize("seed", metafunc.config.getoption("seed"))
diff --git a/tests/resources/ResourceFile_test_convert_to_csv/ResourceFile_test_convert_to_csv.xlsx b/tests/resources/ResourceFile_test_convert_to_csv/ResourceFile_test_convert_to_csv.xlsx
deleted file mode 100644
index 84edbd2636..0000000000
--- a/tests/resources/ResourceFile_test_convert_to_csv/ResourceFile_test_convert_to_csv.xlsx
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:af1a6a6aa24a7de385efdf1564da3e3abfbba9fe467d92212b5c87b127e899f6
-size 10714
diff --git a/tests/resources/probability_premature_death/0/0/tlo.methods.demography.pickle b/tests/resources/probability_premature_death/0/0/tlo.methods.demography.pickle
deleted file mode 100644
index 896ce51bf6..0000000000
--- a/tests/resources/probability_premature_death/0/0/tlo.methods.demography.pickle
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f2cddd2f691393fc27e990170f76ff12a2962d3fbee986deee459a6eb4996fd7
-size 243603
diff --git a/tests/resources/scenario.py b/tests/resources/scenario.py
deleted file mode 100644
index 204c322f11..0000000000
--- a/tests/resources/scenario.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from tlo import Date, logging
-from tlo.scenario import BaseScenario
-
-
-class TestScenario(BaseScenario):
- def __init__(self):
- super().__init__()
- self.seed = 655123742
- self.start_date = Date(2010, 1, 1)
- self.end_date = Date(2011, 1, 1)
- self.pop_size = 2000
- self.number_of_draws = 5
- self.runs_per_draw = 1
-
- def log_configuration(self):
- return {
- 'directory': None,
- 'custom_levels': {
- '*': logging.INFO,
- }
- }
-
- def modules(self):
- return []
-
- def add_arguments(self, parser):
- parser.add_argument('--pop-size', type=int)
-
- def draw_parameters(self, draw_number, rng):
- return {
- 'Lifestyle': {
- 'init_p_urban': rng.randint(10, 20) / 100.0,
- 'init_p_high_sugar': 0.52,
- },
- }
diff --git a/tests/test_alri.py b/tests/test_alri.py
index fcce8b4b42..0fba5fea8d 100644
--- a/tests/test_alri.py
+++ b/tests/test_alri.py
@@ -435,11 +435,7 @@ def __will_die_of_alri(**kwargs):
assert pd.isnull(person['ri_scheduled_death_date'])
# Check that they have some symptoms caused by ALRI
- assert 0 < len(
- sim.modules["SymptomManager"].has_what(
- person_id=person_id, disease_module=sim.modules["Alri"]
- )
- )
+ assert 0 < len(sim.modules['SymptomManager'].has_what(person_id, sim.modules['Alri']))
# Check that there is a AlriNaturalRecoveryEvent scheduled for this person:
recov_event_tuple = [event_tuple for event_tuple in sim.find_events_for_person(person_id) if
@@ -462,11 +458,7 @@ def __will_die_of_alri(**kwargs):
assert pd.isnull(person['ri_scheduled_death_date'])
# check they they have no symptoms:
- assert 0 == len(
- sim.modules["SymptomManager"].has_what(
- person_id=person_id, disease_module=sim.modules["Alri"]
- )
- )
+ assert 0 == len(sim.modules['SymptomManager'].has_what(person_id, sim.modules['Alri']))
# check it's logged (one infection + one recovery)
assert 1 == sim.modules['Alri'].logging_event.trackers['incident_cases'].report_current_total()
diff --git a/tests/test_analysis.py b/tests/test_analysis.py
index 0f42b2d851..2686e431b0 100644
--- a/tests/test_analysis.py
+++ b/tests/test_analysis.py
@@ -1,5 +1,4 @@
import os
-import textwrap
from pathlib import Path
from typing import List
@@ -19,7 +18,6 @@
get_parameters_for_improved_healthsystem_and_healthcare_seeking,
get_parameters_for_status_quo,
get_root_path,
- merge_log_files,
mix_scenarios,
order_of_coarse_appt,
order_of_short_treatment_ids,
@@ -686,99 +684,3 @@ def check_log(log):
sim = Simulation(start_date=Date(2010, 1, 1), seed=seed, log_config=log_config)
check_log(run_simulation_and_cause_one_death(sim))
-
-def test_merge_log_files(tmp_path):
- log_file_path_1 = tmp_path / "log_file_1"
- log_file_path_1.write_text(
- textwrap.dedent(
- """\
- {"uuid": "b07", "type": "header", "module": "m0", "key": "info", "level": "INFO", "columns": {"msg": "str"}, "description": null}
- {"uuid": "b07", "date": "2010-01-01T00:00:00", "values": ["0"]}
- {"uuid": "0b3", "type": "header", "module": "m1", "key": "a", "level": "INFO", "columns": {"msg": "str"}, "description": "A"}
- {"uuid": "0b3", "date": "2010-01-01T00:00:00", "values": ["1"]}
- {"uuid": "ed4", "type": "header", "module": "m2", "key": "b", "level": "INFO", "columns": {"msg": "str"}, "description": "B"}
- {"uuid": "ed4", "date": "2010-01-02T00:00:00", "values": ["2"]}
- {"uuid": "477", "type": "header", "module": "m2", "key": "c", "level": "INFO", "columns": {"msg": "str"}, "description": "C"}
- {"uuid": "477", "date": "2010-01-02T00:00:00", "values": ["3"]}
- {"uuid": "b5c", "type": "header", "module": "m2", "key": "d", "level": "INFO", "columns": {"msg": "str"}, "description": "D"}
- {"uuid": "b5c", "date": "2010-01-03T00:00:00", "values": ["4"]}
- {"uuid": "477", "date": "2010-01-03T00:00:00", "values": ["5"]}
- """
- )
- )
- log_file_path_2 = tmp_path / "log_file_2"
- log_file_path_2.write_text(
- textwrap.dedent(
- """\
- {"uuid": "b07", "type": "header", "module": "m0", "key": "info", "level": "INFO", "columns": {"msg": "str"}, "description": null}
- {"uuid": "b07", "date": "2010-01-04T00:00:00", "values": ["6"]}
- {"uuid": "ed4", "type": "header", "module": "m2", "key": "b", "level": "INFO", "columns": {"msg": "str"}, "description": "B"}
- {"uuid": "ed4", "date": "2010-01-04T00:00:00", "values": ["7"]}
- {"uuid": "ed4", "date": "2010-01-05T00:00:00", "values": ["8"]}
- {"uuid": "0b3", "type": "header", "module": "m1", "key": "a", "level": "INFO", "columns": {"msg": "str"}, "description": "A"}
- {"uuid": "0b3", "date": "2010-01-06T00:00:00", "values": ["9"]}
- {"uuid": "a19", "type": "header", "module": "m3", "key": "e", "level": "INFO", "columns": {"msg": "str"}, "description": "E"}
- {"uuid": "a19", "date": "2010-01-03T00:00:00", "values": ["10"]}
- """
- )
- )
- expected_merged_log_file_content = textwrap.dedent(
- """\
- {"uuid": "b07", "type": "header", "module": "m0", "key": "info", "level": "INFO", "columns": {"msg": "str"}, "description": null}
- {"uuid": "b07", "date": "2010-01-01T00:00:00", "values": ["0"]}
- {"uuid": "0b3", "type": "header", "module": "m1", "key": "a", "level": "INFO", "columns": {"msg": "str"}, "description": "A"}
- {"uuid": "0b3", "date": "2010-01-01T00:00:00", "values": ["1"]}
- {"uuid": "ed4", "type": "header", "module": "m2", "key": "b", "level": "INFO", "columns": {"msg": "str"}, "description": "B"}
- {"uuid": "ed4", "date": "2010-01-02T00:00:00", "values": ["2"]}
- {"uuid": "477", "type": "header", "module": "m2", "key": "c", "level": "INFO", "columns": {"msg": "str"}, "description": "C"}
- {"uuid": "477", "date": "2010-01-02T00:00:00", "values": ["3"]}
- {"uuid": "b5c", "type": "header", "module": "m2", "key": "d", "level": "INFO", "columns": {"msg": "str"}, "description": "D"}
- {"uuid": "b5c", "date": "2010-01-03T00:00:00", "values": ["4"]}
- {"uuid": "477", "date": "2010-01-03T00:00:00", "values": ["5"]}
- {"uuid": "b07", "date": "2010-01-04T00:00:00", "values": ["6"]}
- {"uuid": "ed4", "date": "2010-01-04T00:00:00", "values": ["7"]}
- {"uuid": "ed4", "date": "2010-01-05T00:00:00", "values": ["8"]}
- {"uuid": "0b3", "date": "2010-01-06T00:00:00", "values": ["9"]}
- {"uuid": "a19", "type": "header", "module": "m3", "key": "e", "level": "INFO", "columns": {"msg": "str"}, "description": "E"}
- {"uuid": "a19", "date": "2010-01-03T00:00:00", "values": ["10"]}
- """
- )
- merged_log_file_path = tmp_path / "merged_log_file"
- merge_log_files(log_file_path_1, log_file_path_2, merged_log_file_path)
- merged_log_file_content = merged_log_file_path.read_text()
- assert merged_log_file_content == expected_merged_log_file_content
-
-
-def test_merge_log_files_with_inconsistent_headers_raises(tmp_path):
- log_file_path_1 = tmp_path / "log_file_1"
- log_file_path_1.write_text(
- textwrap.dedent(
- """\
- {"uuid": "b07", "type": "header", "module": "m0", "key": "info", "level": "INFO", "columns": {"msg": "str"}, "description": null}
- {"uuid": "b07", "date": "2010-01-01T00:00:00", "values": ["0"]}
- """
- )
- )
- log_file_path_2 = tmp_path / "log_file_2"
- log_file_path_2.write_text(
- textwrap.dedent(
- """\
- {"uuid": "b07", "type": "header", "module": "m0", "key": "info", "level": "INFO", "columns": {"msg": "int"}, "description": null}
- {"uuid": "b07", "date": "2010-01-04T00:00:00", "values": [1]}
- """
- )
- )
- merged_log_file_path = tmp_path / "merged_log_file"
- with pytest.raises(RuntimeError, match="Inconsistent header lines"):
- merge_log_files(log_file_path_1, log_file_path_2, merged_log_file_path)
-
-
-def test_merge_log_files_inplace_raises(tmp_path):
- log_file_path_1 = tmp_path / "log_file_1"
- log_file_path_1.write_text("foo")
- log_file_path_2 = tmp_path / "log_file_2"
- log_file_path_2.write_text("bar")
- with pytest.raises(ValueError, match="output_path"):
- merge_log_files(log_file_path_1, log_file_path_2, log_file_path_1)
- with pytest.raises(ValueError, match="output_path"):
- merge_log_files(log_file_path_1, log_file_path_2, log_file_path_2)
diff --git a/tests/test_basic_sims.py b/tests/test_basic_sims.py
index ea29aa601f..5200ab730c 100644
--- a/tests/test_basic_sims.py
+++ b/tests/test_basic_sims.py
@@ -184,7 +184,7 @@ def __init__(self, module, end_date):
super().__init__(module=module, frequency=DateOffset(days=1), end_date=end_date)
def apply(self, population):
- population.props.loc[0, 'last_run'] = self.module.sim.date
+ population.props.loc[0, 'last_run'] = population.sim.date
class MyOtherEvent(PopulationScopeEventMixin, RegularEvent):
def __init__(self, module):
diff --git a/tests/test_beddays.py b/tests/test_beddays.py
index 224619e8b3..614719fc86 100644
--- a/tests/test_beddays.py
+++ b/tests/test_beddays.py
@@ -2,7 +2,6 @@
import copy
import os
from pathlib import Path
-from typing import Dict
import pandas as pd
import pytest
@@ -84,88 +83,6 @@ def test_beddays_in_isolation(tmpdir, seed):
assert ([cap_bedtype1] * days_sim == tracker.values).all()
-def test_beddays_allocation_resolution(tmpdir, seed):
- sim = Simulation(start_date=start_date, seed=seed)
- sim.register(
- demography.Demography(resourcefilepath=resourcefilepath),
- healthsystem.HealthSystem(resourcefilepath=resourcefilepath),
- )
-
- # Update BedCapacity data with a simple table:
- level2_facility_ids = [128, 129, 130] # <-- the level 2 facilities for each region
- # This ensures over-allocations have to be properly resolved
- cap_bedtype1 = 10
- cap_bedtype2 = 10
- cap_bedtype3 = 10
-
- # create a simple bed capacity dataframe
- hs = sim.modules["HealthSystem"]
- hs.parameters["BedCapacity"] = pd.DataFrame(
- data={
- "Facility_ID": level2_facility_ids,
- "bedtype1": cap_bedtype1,
- "bedtype2": cap_bedtype2,
- "bedtype3": cap_bedtype3,
- }
- )
-
- sim.make_initial_population(n=100)
- sim.simulate(end_date=start_date)
-
- # reset bed days tracker to the start_date of the simulation
- hs.bed_days.initialise_beddays_tracker()
-
- def assert_footprint_matches_expected(
- footprint: Dict[str, int], expected_footprint: Dict[str, int]
- ):
- """
- Asserts that two footprints are identical.
- The footprint provided as the 2nd argument is assumed to be the footprint
- that we want to match, and the 1st as the result of the program attempting
- to resolve over-allocations.
- """
- assert len(footprint) == len(
- expected_footprint
- ), "Bed type footprints did not return same allocations."
- for bed_type, expected_days in expected_footprint.items():
- allocated_days = footprint[bed_type]
- assert expected_days == allocated_days, (
- f"Bed type {bed_type} was allocated {allocated_days} upon combining, "
- f"but expected it to get {expected_days}."
- )
-
- # Check that combining footprints for a person returns the expected output
-
- # SIMPLE 2-bed days case
- # Test uses example fail case given in https://github.com/UCL/TLOmodel/issues/1399
- # Person p has: bedtyp1 for 2 days, bedtype2 for 0 days.
- # Person p then assigned: bedtype1 for 1 days, bedtype2 for 6 days.
- # EXPECT: p's footprints are combined into bedtype1 for 2 days, bedtype2 for 5 days.
- existing_footprint = {"bedtype1": 2, "bedtype2": 0, "bedtype3": 0}
- incoming_footprint = {"bedtype1": 1, "bedtype2": 6, "bedtype3": 0}
- expected_resolution = {"bedtype1": 2, "bedtype2": 5, "bedtype3": 0}
- allocated_footprint = hs.bed_days.combine_footprints_for_same_patient(
- existing_footprint, incoming_footprint
- )
- assert_footprint_matches_expected(allocated_footprint, expected_resolution)
-
- # TEST case involve 3 different bed-types.
- # Person p has: bedtype1 for 2 days, then bedtype3 for 4 days.
- # p is assigned: bedtype1 for 1 day, bedtype2 for 3 days, and bedtype3 for 1 day.
- # EXPECT: p spends 2 days in each bedtype;
- # - Day 1 needs bedtype1 for both footprints
- # - Day 2 existing footprint at bedtype1 overwrites incoming at bedtype2
- # - Day 3 & 4 incoming footprint at bedtype2 overwrites existing allocation to bedtype3
- # - Day 5 both footprints want bedtype3
- # - Day 6 existing footprint needs bedtype3, whilst incoming footprint is over.s
- existing_footprint = {"bedtype1": 2, "bedtype2": 0, "bedtype3": 4}
- incoming_footprint = {"bedtype1": 1, "bedtype2": 3, "bedtype3": 1}
- expected_resolution = {"bedtype1": 2, "bedtype2": 2, "bedtype3": 2}
- allocated_footprint = hs.bed_days.combine_footprints_for_same_patient(
- existing_footprint, incoming_footprint
- )
- assert_footprint_matches_expected(allocated_footprint, expected_resolution)
-
def check_dtypes(simulation):
# check types of columns
df = simulation.population.props
@@ -1056,82 +973,3 @@ def apply(self, person_id, squeeze_factor):
# Check that the facility_id is included for each entry in the `HSI_Events` log, including HSI Events for
# in-patient appointments.
assert not (log_hsi['Facility_ID'] == -99).any()
-
-def test_beddays_availability_switch(seed):
- """
- Test that calling bed_days.switch_beddays_availability correctly updates the
- bed capacities and adjusts the existing trackers to reflect the new capacities.
- """
- sim = Simulation(start_date=start_date, seed=seed)
- sim.register(
- demography.Demography(resourcefilepath=resourcefilepath),
- healthsystem.HealthSystem(resourcefilepath=resourcefilepath),
- )
-
- # get shortcut to HealthSystem Module
- hs: healthsystem.HealthSystem = sim.modules["HealthSystem"]
-
- # As obtained from the resource file
- facility_id_with_patient = 128
- facility_id_without_patient = 129
- bedtype1_init_capacity = 5
- bedtype2_init_capacity = 10
-
- # Create a simple bed capacity dataframe with capacity designated for two regions
- hs.parameters["BedCapacity"] = pd.DataFrame(
- data={
- "Facility_ID": [
- facility_id_with_patient, #<-- patient 0 is admitted here
- facility_id_without_patient,
- ],
- "bedtype1": bedtype1_init_capacity,
- "bedtype2": bedtype2_init_capacity,
- }
- )
- sim.make_initial_population(n=100)
- sim.simulate(end_date=start_date)
-
- day_2 = start_date + pd.DateOffset(days=1)
- day_3 = start_date + pd.DateOffset(days=2)
- day_4 = start_date + pd.DateOffset(days=3)
-
- bed_days = hs.bed_days
- # Reset the bed occupancies
- bed_days.initialise_beddays_tracker()
- # Have a patient occupy a bed at the start of the simulation
- bed_days.impose_beddays_footprint(person_id=0, footprint={"bedtype1": 3, "bedtype2": 0})
-
- # Have the bed_days availability switch to "none" on the 2nd simulation day
- bed_days.switch_beddays_availability("none", effective_on_and_from=day_2)
-
- # We should now see that the scaled capacities are all zero
- assert (
- not bed_days._scaled_capacity.any().any()
- ), "At least one bed capacity was not set to 0"
- # We should also see that bedtype1 should have -1 beds available for days 2 and 3 of the simulation,
- # due to the existing occupancy and the new capacity of 0.
- # It should have 4 beds available on the first day (since the original capacity was 5 and the availability
- # switch happens day 2).
- # It should then have 0 beds available after (not including) day 3
- bedtype1: pd.DataFrame = bed_days.bed_tracker["bedtype1"]
- bedtype2: pd.DataFrame = bed_days.bed_tracker["bedtype2"]
-
- assert (
- bedtype1.loc[start_date, facility_id_with_patient] == bedtype1_init_capacity - 1
- and bedtype1.loc[start_date, facility_id_without_patient]
- == bedtype1_init_capacity
- ), "Day 1 capacities were incorrectly affected"
- assert (bedtype1.loc[day_2:day_3, facility_id_with_patient] == -1).all() and (
- bedtype1.loc[day_2:day_3, facility_id_without_patient] == 0
- ).all(), "Day 2 & 3 capacities were not updated correctly"
- assert (
- (bedtype1.loc[day_4:, :] == 0).all().all()
- ), "Day 4 onwards did not have correct capacity"
-
- # Bedtype 2 should have also have been updated, but there is no funny business here.
- assert (
- (bedtype2.loc[day_2:, :] == 0).all().all()
- ), "Bedtype 2 was not updated correctly"
- assert (
- (bedtype2.loc[start_date, :] == bedtype2_init_capacity).all().all()
- ), "Bedtype 2 had capacity updated on the incorrect dates"
diff --git a/tests/test_cardiometabolicdisorders.py b/tests/test_cardiometabolicdisorders.py
index 977caa4c91..a40fdad69b 100644
--- a/tests/test_cardiometabolicdisorders.py
+++ b/tests/test_cardiometabolicdisorders.py
@@ -770,7 +770,7 @@ def test_hsi_emergency_events(seed):
assert pd.isnull(df.at[person_id, f'nc_{event}_scheduled_date_death'])
assert isinstance(sim.modules['HealthSystem'].HSI_EVENT_QUEUE[0].hsi_event,
HSI_CardioMetabolicDisorders_StartWeightLossAndMedication)
- assert f"{event}_damage" not in sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert f"{event}_damage" not in sim.modules['SymptomManager'].has_what(person_id)
def test_no_availability_of_consumables_for_conditions(seed):
diff --git a/tests/test_copd.py b/tests/test_copd.py
index b47d803529..6c8b8a0917 100644
--- a/tests/test_copd.py
+++ b/tests/test_copd.py
@@ -211,12 +211,12 @@ def test_moderate_exacerbation():
df.at[person_id, 'ch_has_inhaler'] = False
# check individuals do not have symptoms before an event is run
- assert 'breathless_moderate' not in sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert 'breathless_moderate' not in sim.modules['SymptomManager'].has_what(person_id)
# run Copd Exacerbation event on an individual and confirm they now have a
# non-emergency symptom(breathless moderate)
copd.CopdExacerbationEvent(copd_module, person_id, severe=False).run()
- assert 'breathless_moderate' in sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert 'breathless_moderate' in sim.modules['SymptomManager'].has_what(person_id)
# Run health seeking behavior event and check non-emergency care is sought
hsp = HealthSeekingBehaviourPoll(sim.modules['HealthSeekingBehaviour'])
@@ -259,15 +259,13 @@ def test_severe_exacerbation():
df.at[person_id, 'ch_has_inhaler'] = False
# check an individual do not have emergency symptoms before an event is run
- assert 'breathless_severe' not in sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert 'breathless_severe' not in sim.modules['SymptomManager'].has_what(person_id)
# schedule exacerbations event setting severe to True. This will ensure the individual has severe exacerbation
copd.CopdExacerbationEvent(copd_module, person_id, severe=True).run()
# severe exacerbation should lead to severe symptom(breathless severe in this case). check this is true
- assert "breathless_severe" in sim.modules["SymptomManager"].has_what(
- person_id=person_id, disease_module=copd_module
- )
+ assert 'breathless_severe' in sim.modules['SymptomManager'].has_what(person_id, copd_module)
# # Run health seeking behavior event and check emergency care is sought
hsp = HealthSeekingBehaviourPoll(module=sim.modules['HealthSeekingBehaviour'])
@@ -422,15 +420,13 @@ def test_referral_logic():
df.at[person_id, 'ch_has_inhaler'] = False
# check an individual do not have emergency symptoms before an event is run
- assert 'breathless_severe' not in sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert 'breathless_severe' not in sim.modules['SymptomManager'].has_what(person_id)
# schedule exacerbations event setting severe to True. This will ensure the individual has severe exacerbation
copd.CopdExacerbationEvent(copd_module, person_id, severe=True).run()
# severe exacerbation should lead to severe symptom(breathless severe in this case). check this is true
- assert "breathless_severe" in sim.modules["SymptomManager"].has_what(
- person_id=person_id, disease_module=copd_module
- )
+ assert 'breathless_severe' in sim.modules['SymptomManager'].has_what(person_id, copd_module)
# Run health seeking behavior event and check emergency care is sought
hsp = HealthSeekingBehaviourPoll(module=sim.modules['HealthSeekingBehaviour'])
diff --git a/tests/test_diarrhoea.py b/tests/test_diarrhoea.py
index 3a4daebc5d..1e43550541 100644
--- a/tests/test_diarrhoea.py
+++ b/tests/test_diarrhoea.py
@@ -403,7 +403,7 @@ def test_do_when_presentation_with_diarrhoea_severe_dehydration(seed):
generic_hsi = HSI_GenericNonEmergencyFirstAppt(
module=sim.modules["HealthSeekingBehaviour"], person_id=person_id
)
- symptoms = {"diarrhoea"}
+ patient_details = sim.population.row_in_readonly_form(person_id)
def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False):
return generic_hsi.healthcare_system.dx_manager.run_dx_test(
@@ -415,14 +415,11 @@ def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False):
sim.modules['HealthSystem'].reset_queue()
sim.modules['Diarrhoea'].parameters['prob_hospitalization_on_danger_signs'] = 1.0
- with sim.population.individual_properties(person_id) as individual_properties:
- sim.modules["Diarrhoea"].do_at_generic_first_appt(
- person_id=person_id,
- individual_properties=individual_properties,
- schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event,
- diagnosis_function=diagnosis_fn,
- symptoms=symptoms,
- )
+ sim.modules["Diarrhoea"].do_at_generic_first_appt(
+ patient_id=person_id,
+ patient_details=patient_details,
+ diagnosis_function=diagnosis_fn,
+ )
evs = sim.modules['HealthSystem'].find_events_for_person(person_id)
assert 1 == len(evs)
@@ -431,14 +428,11 @@ def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False):
# 2) If DxTest of danger signs perfect but 0% chance of referral --> Inpatient HSI should not be created
sim.modules['HealthSystem'].reset_queue()
sim.modules['Diarrhoea'].parameters['prob_hospitalization_on_danger_signs'] = 0.0
- with sim.population.individual_properties(person_id) as individual_properties:
- sim.modules["Diarrhoea"].do_at_generic_first_appt(
- person_id=person_id,
- individual_properties=individual_properties,
- schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event,
- diagnosis_function=diagnosis_fn,
- symptoms=symptoms,
- )
+ sim.modules["Diarrhoea"].do_at_generic_first_appt(
+ patient_id=person_id,
+ patient_details=patient_details,
+ diagnosis_function=diagnosis_fn,
+ )
evs = sim.modules['HealthSystem'].find_events_for_person(person_id)
assert 1 == len(evs)
assert isinstance(evs[0][1], HSI_Diarrhoea_Treatment_Outpatient)
@@ -498,7 +492,7 @@ def test_do_when_presentation_with_diarrhoea_severe_dehydration_dxtest_notfuncti
df.loc[person_id, props_new.keys()] = props_new.values()
generic_hsi = HSI_GenericNonEmergencyFirstAppt(
module=sim.modules['HealthSeekingBehaviour'], person_id=person_id)
- symptoms = {"diarrhoea"}
+ patient_details = sim.population.row_in_readonly_form(person_id)
def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False):
return generic_hsi.healthcare_system.dx_manager.run_dx_test(
@@ -511,14 +505,11 @@ def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False):
# Only an out-patient appointment should be created as the DxTest for danger signs is not functional.
sim.modules['Diarrhoea'].parameters['prob_hospitalization_on_danger_signs'] = 0.0
sim.modules['HealthSystem'].reset_queue()
- with sim.population.individual_properties(person_id) as individual_properties:
- sim.modules["Diarrhoea"].do_at_generic_first_appt(
- person_id=person_id,
- individual_properties=individual_properties,
- schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event,
- diagnosis_function=diagnosis_fn,
- symptoms=symptoms,
- )
+ sim.modules["Diarrhoea"].do_at_generic_first_appt(
+ patient_id=person_id,
+ patient_details=patient_details,
+ diagnosis_function=diagnosis_fn,
+ )
evs = sim.modules['HealthSystem'].find_events_for_person(person_id)
assert 1 == len(evs)
assert isinstance(evs[0][1], HSI_Diarrhoea_Treatment_Outpatient)
@@ -577,7 +568,7 @@ def test_do_when_presentation_with_diarrhoea_non_severe_dehydration(seed):
df.loc[person_id, props_new.keys()] = props_new.values()
generic_hsi = HSI_GenericNonEmergencyFirstAppt(
module=sim.modules['HealthSeekingBehaviour'], person_id=person_id)
- symptoms = {"diarrhoea"}
+ patient_details = sim.population.row_in_readonly_form(person_id)
def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False):
return generic_hsi.healthcare_system.dx_manager.run_dx_test(
@@ -588,14 +579,9 @@ def diagnosis_fn(tests, use_dict: bool = False, report_tried: bool = False):
)
# 1) Outpatient HSI should be created
sim.modules["HealthSystem"].reset_queue()
- with sim.population.individual_properties(person_id) as individual_properties:
- sim.modules["Diarrhoea"].do_at_generic_first_appt(
- person_id=person_id,
- individual_properties=individual_properties,
- schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event,
- diagnosis_function=diagnosis_fn,
- symptoms=symptoms,
- )
+ sim.modules["Diarrhoea"].do_at_generic_first_appt(
+ patient_id=person_id, patient_details=patient_details, diagnosis_function=diagnosis_fn
+ )
evs = sim.modules["HealthSystem"].find_events_for_person(person_id)
assert 1 == len(evs)
diff --git a/tests/test_equipment.py b/tests/test_equipment.py
index e7b8f03ccc..a02ea282f8 100644
--- a/tests/test_equipment.py
+++ b/tests/test_equipment.py
@@ -1,6 +1,5 @@
"""This file contains all the tests to do with Equipment."""
import os
-from ast import literal_eval
from pathlib import Path
from typing import Dict
@@ -23,18 +22,14 @@ def test_core_functionality_of_equipment_class(seed):
# Create toy data
catalogue = pd.DataFrame(
- # PkgWith0+1 stands alone or as multiple pkgs for one item; PkgWith1 is only as multiple pkgs
- # for one item; PkgWith3 only stands alone
[
{"Item_Description": "ItemZero", "Item_Code": 0, "Pkg_Name": 'PkgWith0+1'},
- {"Item_Description": "ItemOne", "Item_Code": 1, "Pkg_Name": 'PkgWith0+1, PkgWith1'},
+ {"Item_Description": "ItemOne", "Item_Code": 1, "Pkg_Name": 'PkgWith0+1'},
{"Item_Description": "ItemTwo", "Item_Code": 2, "Pkg_Name": float('nan')},
- {"Item_Description": "ItemThree", "Item_Code": 3, "Pkg_Name": 'PkgWith3'},
]
)
data_availability = pd.DataFrame(
- # item 0 is not available anywhere; item 1 is available everywhere; item 2 is available only at facility_id=1;
- # item 3 is available only at facility_id=0
+ # item 0 is not available anywhere; item 1 is available everywhere; item 2 is available only at facility_id=1
[
{"Item_Code": 0, "Facility_ID": 0, "Pr_Available": 0.0},
{"Item_Code": 0, "Facility_ID": 1, "Pr_Available": 0.0},
@@ -42,8 +37,6 @@ def test_core_functionality_of_equipment_class(seed):
{"Item_Code": 1, "Facility_ID": 1, "Pr_Available": 1.0},
{"Item_Code": 2, "Facility_ID": 0, "Pr_Available": 0.0},
{"Item_Code": 2, "Facility_ID": 1, "Pr_Available": 1.0},
- {"Item_Code": 3, "Facility_ID": 0, "Pr_Available": 1.0},
- {"Item_Code": 3, "Facility_ID": 1, "Pr_Available": 0.0},
]
)
mfl = pd.DataFrame(
@@ -83,22 +76,6 @@ def test_core_functionality_of_equipment_class(seed):
with pytest.warns():
eq_default.parse_items('ItemThatIsNotDefined')
- # Lookup the item_codes that belong in a particular package.
- # - When package is recognised
- # if items are in the same package (once standing alone, once within multiple pkgs defined for item)
- assert {0, 1} == eq_default.from_pkg_names(pkg_names='PkgWith0+1')
- # if the pkg within multiple pkgs defined for item
- assert {1} == eq_default.from_pkg_names(pkg_names='PkgWith1')
- # if the pkg only stands alone
- assert {3} == eq_default.from_pkg_names(pkg_names='PkgWith3')
- # Lookup the item_codes that belong to multiple specified packages.
- assert {0, 1, 3} == eq_default.from_pkg_names(pkg_names={'PkgWith0+1', 'PkgWith3'})
- assert {1, 3} == eq_default.from_pkg_names(pkg_names={'PkgWith1', 'PkgWith3'})
-
- # - When package is not recognised (should raise an error)
- with pytest.raises(ValueError):
- eq_default.from_pkg_names(pkg_names='')
-
# Testing checking on available of items
# - calling when all items available (should be true)
assert eq_default.is_all_items_available(item_codes={1, 2}, facility_id=1)
@@ -155,11 +132,19 @@ def test_core_functionality_of_equipment_class(seed):
# - Check that internal record is as expected
assert {0: {0: 1, 1: 2}, 1: {0: 1, 1: 1}} == dict(eq_default._record_of_equipment_used_by_facility_id)
+ # Lookup the item_codes that belong in a particular package.
+ # - When package is recognised
+ assert {0, 1} == eq_default.lookup_item_codes_from_pkg_name(pkg_name='PkgWith0+1') # these items are in the same
+ # package
+ # - Error thrown when package is not recognised
+ with pytest.raises(ValueError):
+ eq_default.lookup_item_codes_from_pkg_name(pkg_name='')
+
+
equipment_item_code_that_is_available = [0, 1, ]
equipment_item_code_that_is_not_available = [2, 3,]
-
def run_simulation_and_return_log(
seed, tmpdir, equipment_in_init, equipment_in_apply
) -> Dict:
@@ -260,7 +245,7 @@ def all_equipment_ever_used(log: Dict) -> set:
(at any facility)."""
s = set()
for i in log["EquipmentEverUsed_ByFacilityID"]['EquipmentEverUsed']:
- s.update(literal_eval(i))
+ s.update(eval(i))
return s
# * An HSI that declares no use of any equipment (logs should be empty).
@@ -475,7 +460,7 @@ def initialise_simulation(self, sim):
# Read log to find what equipment used
df = parse_log_file(sim.log_filepath)["tlo.methods.healthsystem.summary"]['EquipmentEverUsed_ByFacilityID']
df = df.drop(index=df.index[~df['Facility_Level'].isin(item_code_needed_at_each_level.keys())])
- df['EquipmentEverUsed'] = df['EquipmentEverUsed'].apply(literal_eval)
+ df['EquipmentEverUsed'] = df['EquipmentEverUsed'].apply(eval).apply(list)
# Check that equipment used at each level matches expectations
assert item_code_needed_at_each_level == df.groupby('Facility_Level')['EquipmentEverUsed'].sum().apply(set).to_dict()
diff --git a/tests/test_hiv.py b/tests/test_hiv.py
index 5a27cf2c33..47ef0d2083 100644
--- a/tests/test_hiv.py
+++ b/tests/test_hiv.py
@@ -224,7 +224,7 @@ def test_generation_of_natural_history_process_no_art(seed):
# run the AIDS onset event for this person:
aids_event.apply(person_id)
- assert "aids_symptoms" in sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert "aids_symptoms" in sim.modules['SymptomManager'].has_what(person_id)
# find the AIDS death event for this person
date_aids_death_event, aids_death_event = \
@@ -274,7 +274,7 @@ def test_generation_of_natural_history_process_with_art_before_aids(seed):
assert [] == [ev for ev in sim.find_events_for_person(person_id) if isinstance(ev[1], hiv.HivAidsDeathEvent)]
# check no AIDS symptoms for this person
- assert "aids_symptoms" not in sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert "aids_symptoms" not in sim.modules['SymptomManager'].has_what(person_id)
def test_generation_of_natural_history_process_with_art_after_aids(seed):
@@ -312,7 +312,7 @@ def test_generation_of_natural_history_process_with_art_after_aids(seed):
date_aids_death_event, aids_death_event = \
[ev for ev in sim.find_events_for_person(person_id) if isinstance(ev[1], hiv.HivAidsDeathEvent)][0]
assert date_aids_death_event > sim.date
- assert "aids_symptoms" in sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert "aids_symptoms" in sim.modules['SymptomManager'].has_what(person_id)
# Put the person on ART with VL suppression prior to the AIDS death (but following AIDS onset)
df.at[person_id, 'hv_art'] = "on_VL_suppressed"
@@ -516,7 +516,7 @@ def test_aids_symptoms_lead_to_treatment_being_initiated(seed):
aids_event.apply(person_id)
# Confirm that they have aids symptoms and an AIDS death schedule
- assert 'aids_symptoms' in sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert 'aids_symptoms' in sim.modules['SymptomManager'].has_what(person_id)
assert 1 == len(
[ev[0] for ev in sim.find_events_for_person(person_id) if isinstance(ev[1], hiv.HivAidsTbDeathEvent)])
diff --git a/tests/test_htm_scaleup.py b/tests/test_htm_scaleup.py
deleted file mode 100644
index fcb538f19c..0000000000
--- a/tests/test_htm_scaleup.py
+++ /dev/null
@@ -1,210 +0,0 @@
-""" Tests for setting up the HIV, TB and malaria scenarios used for projections """
-
-import os
-from pathlib import Path
-
-import pandas as pd
-
-from tlo import Date, Simulation
-from tlo.methods import (
- demography,
- enhanced_lifestyle,
- epi,
- healthburden,
- healthseekingbehaviour,
- healthsystem,
- hiv,
- malaria,
- simplified_births,
- symptommanager,
- tb,
-)
-
-resourcefilepath = Path(os.path.dirname(__file__)) / "../resources"
-
-start_date = Date(2010, 1, 1)
-scaleup_start_year = 2012 # <-- the scale-up will occur on 1st January of that year
-end_date = Date(2013, 1, 1)
-
-
-def get_sim(seed):
- """
- register all necessary modules for the tests to run
- """
-
- sim = Simulation(start_date=start_date, seed=seed)
-
- # Register the appropriate modules
- sim.register(
- demography.Demography(resourcefilepath=resourcefilepath),
- simplified_births.SimplifiedBirths(resourcefilepath=resourcefilepath),
- enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath),
- healthsystem.HealthSystem(resourcefilepath=resourcefilepath),
- symptommanager.SymptomManager(resourcefilepath=resourcefilepath),
- healthseekingbehaviour.HealthSeekingBehaviour(resourcefilepath=resourcefilepath),
- healthburden.HealthBurden(resourcefilepath=resourcefilepath),
- epi.Epi(resourcefilepath=resourcefilepath),
- hiv.Hiv(resourcefilepath=resourcefilepath),
- tb.Tb(resourcefilepath=resourcefilepath),
- malaria.Malaria(resourcefilepath=resourcefilepath),
- )
-
- return sim
-
-
-def check_initial_params(sim):
-
- original_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name='parameters')
-
- # check initial parameters
- assert sim.modules["Hiv"].parameters["beta"] == \
- original_params.loc[original_params.parameter_name == "beta", "value"].values[0]
- assert sim.modules["Hiv"].parameters["prob_prep_for_fsw_after_hiv_test"] == original_params.loc[
- original_params.parameter_name == "prob_prep_for_fsw_after_hiv_test", "value"].values[0]
- assert sim.modules["Hiv"].parameters["prob_prep_for_agyw"] == original_params.loc[
- original_params.parameter_name == "prob_prep_for_agyw", "value"].values[0]
- assert sim.modules["Hiv"].parameters["probability_of_being_retained_on_prep_every_3_months"] == original_params.loc[
- original_params.parameter_name == "probability_of_being_retained_on_prep_every_3_months", "value"].values[0]
- assert sim.modules["Hiv"].parameters["prob_circ_after_hiv_test"] == original_params.loc[
- original_params.parameter_name == "prob_circ_after_hiv_test", "value"].values[0]
-
-
-def test_hiv_scale_up(seed):
- """ test hiv program scale-up changes parameters correctly
- and on correct date """
-
- original_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name="parameters")
- new_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name="scaleup_parameters")
-
- popsize = 100
-
- sim = get_sim(seed=seed)
-
- # check initial parameters
- check_initial_params(sim)
-
- # update parameters to instruct there to be a scale-up
- sim.modules["Hiv"].parameters["type_of_scaleup"] = 'target'
- sim.modules["Hiv"].parameters["scaleup_start_year"] = scaleup_start_year
-
- # Make the population
- sim.make_initial_population(n=popsize)
- sim.simulate(end_date=end_date)
-
- # check HIV parameters changed
- assert sim.modules["Hiv"].parameters["beta"] < original_params.loc[
- original_params.parameter_name == "beta", "value"].values[0]
- assert sim.modules["Hiv"].parameters["prob_prep_for_fsw_after_hiv_test"] == new_params.loc[
- new_params.parameter == "prob_prep_for_fsw_after_hiv_test", "target_value"].values[0]
- assert sim.modules["Hiv"].parameters["prob_prep_for_agyw"] == new_params.loc[
- new_params.parameter == "prob_prep_for_agyw", "target_value"].values[0]
- assert sim.modules["Hiv"].parameters["probability_of_being_retained_on_prep_every_3_months"] == new_params.loc[
- new_params.parameter == "probability_of_being_retained_on_prep_every_3_months", "target_value"].values[0]
- assert sim.modules["Hiv"].parameters["prob_circ_after_hiv_test"] == new_params.loc[
- new_params.parameter == "prob_circ_after_hiv_test", "target_value"].values[0]
-
- # check malaria parameters unchanged
- mal_original_params = pd.read_excel(resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx',
- sheet_name="parameters")
- mal_rdt_testing = pd.read_excel(resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx',
- sheet_name="WHO_TestData2023")
-
- assert sim.modules["Malaria"].parameters["prob_malaria_case_tests"] == mal_original_params.loc[
- mal_original_params.parameter_name == "prob_malaria_case_tests", "value"].values[0]
- pd.testing.assert_series_equal(sim.modules["Malaria"].parameters["rdt_testing_rates"]["Rate_rdt_testing"],
- mal_rdt_testing["Rate_rdt_testing"])
-
- # all irs coverage levels should be < 1.0
- assert sim.modules["Malaria"].itn_irs['irs_rate'].all() < 1.0
- # itn rates for 2019 onwards
- assert sim.modules["Malaria"].parameters["itn"] == mal_original_params.loc[
- mal_original_params.parameter_name == "itn", "value"].values[0]
-
- # check tb parameters unchanged
- tb_original_params = pd.read_excel(resourcefilepath / 'ResourceFile_TB.xlsx', sheet_name="parameters")
- tb_testing = pd.read_excel(resourcefilepath / 'ResourceFile_TB.xlsx', sheet_name="NTP2019")
-
- pd.testing.assert_series_equal(sim.modules["Tb"].parameters["rate_testing_active_tb"]["treatment_coverage"],
- tb_testing["treatment_coverage"])
- assert sim.modules["Tb"].parameters["prob_tx_success_ds"] == tb_original_params.loc[
- tb_original_params.parameter_name == "prob_tx_success_ds", "value"].values[0]
- assert sim.modules["Tb"].parameters["prob_tx_success_mdr"] == tb_original_params.loc[
- tb_original_params.parameter_name == "prob_tx_success_mdr", "value"].values[0]
- assert sim.modules["Tb"].parameters["prob_tx_success_0_4"] == tb_original_params.loc[
- tb_original_params.parameter_name == "prob_tx_success_0_4", "value"].values[0]
- assert sim.modules["Tb"].parameters["prob_tx_success_5_14"] == tb_original_params.loc[
- tb_original_params.parameter_name == "prob_tx_success_5_14", "value"].values[0]
- assert sim.modules["Tb"].parameters["first_line_test"] == tb_original_params.loc[
- tb_original_params.parameter_name == "first_line_test", "value"].values[0]
-
-
-def test_htm_scale_up(seed):
- """ test hiv/tb/malaria program scale-up changes parameters correctly
- and on correct date """
-
- # Load data on HIV prevalence
- original_hiv_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name="parameters")
- new_hiv_params = pd.read_excel(resourcefilepath / 'ResourceFile_HIV.xlsx', sheet_name="scaleup_parameters")
-
- popsize = 100
-
- sim = get_sim(seed=seed)
-
- # check initial parameters
- check_initial_params(sim)
-
- # update parameters
- sim.modules["Hiv"].parameters["type_of_scaleup"] = 'target'
- sim.modules["Hiv"].parameters["scaleup_start_year"] = scaleup_start_year
- sim.modules["Tb"].parameters["type_of_scaleup"] = 'target'
- sim.modules["Tb"].parameters["scaleup_start_year"] = scaleup_start_year
- sim.modules["Malaria"].parameters["type_of_scaleup"] = 'target'
- sim.modules["Malaria"].parameters["scaleup_start_year"] = scaleup_start_year
-
- # Make the population
- sim.make_initial_population(n=popsize)
- sim.simulate(end_date=end_date)
-
- # check HIV parameters changed
- assert sim.modules["Hiv"].parameters["beta"] < original_hiv_params.loc[
- original_hiv_params.parameter_name == "beta", "value"].values[0]
- assert sim.modules["Hiv"].parameters["prob_prep_for_fsw_after_hiv_test"] == new_hiv_params.loc[
- new_hiv_params.parameter == "prob_prep_for_fsw_after_hiv_test", "target_value"].values[0]
- assert sim.modules["Hiv"].parameters["prob_prep_for_agyw"] == new_hiv_params.loc[
- new_hiv_params.parameter == "prob_prep_for_agyw", "target_value"].values[0]
- assert sim.modules["Hiv"].parameters["probability_of_being_retained_on_prep_every_3_months"] == new_hiv_params.loc[
- new_hiv_params.parameter == "probability_of_being_retained_on_prep_every_3_months", "target_value"].values[0]
- assert sim.modules["Hiv"].parameters["prob_circ_after_hiv_test"] == new_hiv_params.loc[
- new_hiv_params.parameter == "prob_circ_after_hiv_test", "target_value"].values[0]
-
- # check malaria parameters changed
- new_mal_params = pd.read_excel(resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx',
- sheet_name="scaleup_parameters")
-
- assert sim.modules["Malaria"].parameters["prob_malaria_case_tests"] == new_mal_params.loc[
- new_mal_params.parameter == "prob_malaria_case_tests", "target_value"].values[0]
- assert sim.modules["Malaria"].parameters["rdt_testing_rates"]["Rate_rdt_testing"].eq(new_mal_params.loc[
- new_mal_params.parameter == "rdt_testing_rates", "target_value"].values[0]).all()
-
- # some irs coverage levels should now = 1.0
- assert sim.modules["Malaria"].itn_irs['irs_rate'].any() == 1.0
- # itn rates for 2019 onwards
- assert sim.modules["Malaria"].parameters["itn"] == new_mal_params.loc[
- new_mal_params.parameter == "itn", "target_value"].values[0]
-
- # check tb parameters changed
- new_tb_params = pd.read_excel(resourcefilepath / 'ResourceFile_TB.xlsx', sheet_name="scaleup_parameters")
-
- assert sim.modules["Tb"].parameters["rate_testing_active_tb"]["treatment_coverage"].eq(new_tb_params.loc[
- new_tb_params.parameter == "tb_treatment_coverage", "target_value"].values[0]).all()
- assert sim.modules["Tb"].parameters["prob_tx_success_ds"] == new_tb_params.loc[
- new_tb_params.parameter == "tb_prob_tx_success_ds", "target_value"].values[0]
- assert sim.modules["Tb"].parameters["prob_tx_success_mdr"] == new_tb_params.loc[
- new_tb_params.parameter == "tb_prob_tx_success_mdr", "target_value"].values[0]
- assert sim.modules["Tb"].parameters["prob_tx_success_0_4"] == new_tb_params.loc[
- new_tb_params.parameter == "tb_prob_tx_success_0_4", "target_value"].values[0]
- assert sim.modules["Tb"].parameters["prob_tx_success_5_14"] == new_tb_params.loc[
- new_tb_params.parameter == "tb_prob_tx_success_5_14", "target_value"].values[0]
- assert sim.modules["Tb"].parameters["first_line_test"] == new_tb_params.loc[
- new_tb_params.parameter == "first_line_test", "target_value"].values[0]
-
diff --git a/tests/test_life_expectancy.py b/tests/test_life_expectancy.py
index 0a77f02310..2465580f65 100644
--- a/tests/test_life_expectancy.py
+++ b/tests/test_life_expectancy.py
@@ -1,15 +1,10 @@
import datetime
import os
-import pickle
from pathlib import Path
-import numpy as np
import pandas as pd
-from tlo.analysis.life_expectancy import (
- get_life_expectancy_estimates,
- get_probability_of_premature_death,
-)
+from tlo.analysis.life_expectancy import get_life_expectancy_estimates
def test_get_life_expectancy():
@@ -38,57 +33,3 @@ def test_get_life_expectancy():
assert sorted(rtn_full.index.to_list()) == ["F", "M"]
assert list(rtn_full.columns.names) == ['draw', 'run']
assert rtn_full.columns.levels[1].to_list() == [0, 1]
-
-
-def test_probability_premature_death(tmpdir, age_before_which_death_is_defined_as_premature: int = 70):
- """
- Test the calculation of the probability of premature death from a simulated cohort.
-
- This function loads results from a dummy cohort (N = 100, with 37 F and 63 M) simulation where all individuals start
- at age 0. The simulation was then run for 70 years (2010 - 2080), during which individuals could die but nobody
- could be born. In this dummy data set, 6 F die and 23 M die prematurely, giving a probability of premature death as
- 0.16 and 0.37, respectively. The premature deaths amongst these individuals is then the number that have died
- before the age of 70 (default value).
- This test uses the calculates the probability of premature death separately for males and females using the
- data from this simulated run and the function get_probability_of_premature_death.
- It then compares these simulated probabilities against the total number of deaths before the age of 70 (default)
- that occurred in the simulated cohort.
- """
- # load results from a dummy cohort where everyone starts at age 0.
- target_period = (datetime.date(2010, 1, 1), datetime.date(2080, 12, 31))
-
- results_folder_dummy_results = Path(os.path.dirname(__file__)) / 'resources' / 'probability_premature_death'
- pickled_file = os.path.join(results_folder_dummy_results, '0', '0', 'tlo.methods.demography.pickle')
-
- # - Compute 'manually' from raw data
- with open(pickled_file, 'rb') as file:
- demography_data = pickle.load(file)
- initial_popsize = {'F': demography_data['population']['female'][0], 'M': demography_data['population']['male'][0]}
- deaths_total = demography_data['death'][['sex', 'age']]
- num_premature_deaths = deaths_total.loc[deaths_total['age'] < age_before_which_death_is_defined_as_premature] \
- .groupby('sex') \
- .size() \
- .to_dict()
- prob_premature_death = {s: num_premature_deaths[s] / initial_popsize[s] for s in ("M", "F")}
-
- # - Compute using utility function
- probability_premature_death_summary = get_probability_of_premature_death(
- results_folder=results_folder_dummy_results,
- target_period=target_period,
- summary=True,
- )
-
- # Confirm both methods gives the same answer
- # (Absolute tolerance of this test is reasonably large (1%) as small assumptions made in the calculation of the
- # cumulative probability of death in each age-group mean that the manual computation done here and the calculation
- # performed in the utility function are not expected to agree perfectly.)
- assert np.isclose(
- probability_premature_death_summary.loc["F"].loc[(0, 'mean')],
- prob_premature_death['F'],
- atol=0.01
- )
- assert np.isclose(
- probability_premature_death_summary.loc["M"].loc[(0, 'mean')],
- prob_premature_death['M'],
- atol=0.01
- )
diff --git a/tests/test_logging.py b/tests/test_logging.py
index 6d094623c4..13151c8be5 100644
--- a/tests/test_logging.py
+++ b/tests/test_logging.py
@@ -1,587 +1,173 @@
-import contextlib
import json
-import logging as _logging
-import sys
-from collections.abc import Generator, Iterable, Mapping
-from itertools import chain, product, repeat
+import os
from pathlib import Path
-from typing import Callable
-import numpy as np
import pandas as pd
import pytest
-import tlo.logging as logging
-import tlo.logging.core as core
-
-
-def _single_row_dataframe(data: dict) -> pd.DataFrame:
- # Single row dataframe 'type' which allows construction by calling on a dictionary
- # of scalars by using an explicit length 1 index while also giving a readable
- # test parameter identifier
- return pd.DataFrame(data, index=[0])
-
-
-LOGGING_LEVELS = [logging.DEBUG, logging.INFO, logging.WARNING, logging.CRITICAL]
-CATCH_ALL_LEVEL = -1
-STRING_DATA_VALUES = ["foo", "bar", "spam"]
-ITERABLE_DATA_VALUES = [(1, 2), (3, 1, 2), ("d", "e"), ("a", "c", 1)]
-MAPPING_DATA_VALUES = [{"a": 1, "b": "spam", 2: None}, {"eggs": "foo", "bar": 1.25}]
-SUPPORTED_SEQUENCE_TYPES = [list, tuple, pd.Series]
-SUPPORTED_ITERABLE_TYPES = SUPPORTED_SEQUENCE_TYPES + [set]
-SUPPORTED_MAPPING_TYPES = [dict, _single_row_dataframe]
-LOGGER_NAMES = ["tlo", "tlo.methods"]
-SIMULATION_DATE = "2010-01-01T00:00:00"
-
-
-class UpdateableSimulateDateGetter:
-
- def __init__(self, start_date=pd.Timestamp(2010, 1, 1)):
- self._date = start_date
-
- def increment_date(self, days=1) -> None:
- self._date += pd.DateOffset(days=days)
-
- def __call__(self) -> str:
- return self._date.isoformat()
-
-
-@pytest.fixture
-def simulation_date_getter() -> core.SimulationDateGetter:
- return lambda: SIMULATION_DATE
-
-
-@pytest.fixture
-def root_level() -> core.LogLevel:
- return logging.WARNING
-
-
-@pytest.fixture
-def stdout_handler_level() -> core.LogLevel:
- return logging.DEBUG
-
-
-@pytest.fixture
-def add_stdout_handler() -> bool:
- return False
-
-
-@pytest.fixture(autouse=True)
-def initialise_logging(
- add_stdout_handler: bool,
- simulation_date_getter: core.SimulationDateGetter,
- root_level: core.LogLevel,
- stdout_handler_level: core.LogLevel,
-) -> Generator[None, None, None]:
- logging.initialise(
- add_stdout_handler=add_stdout_handler,
- simulation_date_getter=simulation_date_getter,
- root_level=root_level,
- stdout_handler_level=stdout_handler_level,
- )
- yield
- logging.reset()
-
-
-@pytest.mark.parametrize("add_stdout_handler", [True, False])
-@pytest.mark.parametrize("root_level", LOGGING_LEVELS, ids=_logging.getLevelName)
-@pytest.mark.parametrize(
- "stdout_handler_level", LOGGING_LEVELS, ids=_logging.getLevelName
-)
-def test_initialise_logging(
- add_stdout_handler: bool,
- simulation_date_getter: core.SimulationDateGetter,
- root_level: core.LogLevel,
- stdout_handler_level: core.LogLevel,
-) -> None:
- logger = logging.getLogger("tlo")
- assert logger.level == root_level
- if add_stdout_handler:
- assert len(logger.handlers) == 1
- handler = logger.handlers[0]
- assert isinstance(handler, _logging.StreamHandler)
- assert handler.stream is sys.stdout
- assert handler.level == stdout_handler_level
- else:
- assert len(logger.handlers) == 0
- assert core._get_simulation_date is simulation_date_getter
-
-
-def _check_handlers(
- logger: core.Logger, expected_number_handlers: int, expected_log_path: Path
-) -> None:
- assert len(logger.handlers) == expected_number_handlers
- file_handlers = [h for h in logger.handlers if isinstance(h, _logging.FileHandler)]
- assert len(file_handlers) == 1
- assert file_handlers[0].baseFilename == str(expected_log_path)
-
-
-@pytest.mark.parametrize("add_stdout_handler", [True, False])
-def test_set_output_file(add_stdout_handler: bool, tmp_path: Path) -> None:
- log_path_1 = tmp_path / "test-1.log"
- log_path_2 = tmp_path / "test-2.log"
- logging.set_output_file(log_path_1)
- logger = logging.getLogger("tlo")
- expected_number_handlers = 2 if add_stdout_handler else 1
- _check_handlers(logger, expected_number_handlers, log_path_1)
- # Setting output file a second time should replace previous file handler rather
- # than add an additional handler and keep existing
- logging.set_output_file(log_path_2)
- _check_handlers(logger, expected_number_handlers, log_path_2)
-
-
-@pytest.mark.parametrize("logger_name", ["tlo", "tlo.methods"])
-def test_getLogger(logger_name: str) -> None:
- logger = logging.getLogger(logger_name)
- assert logger.name == logger_name
- assert isinstance(logger.handlers, list)
- assert isinstance(logger.level, int)
- assert logger.isEnabledFor(logger.level)
- assert logging.getLogger(logger_name) is logger
-
-
-@pytest.mark.parametrize("logger_name", ["foo", "spam.tlo"])
-def test_getLogger_invalid_name_raises(logger_name: str) -> None:
- with pytest.raises(AssertionError, match=logger_name):
- logging.getLogger(logger_name)
-
-
-@pytest.mark.parametrize("mapping_data", MAPPING_DATA_VALUES)
-@pytest.mark.parametrize("mapping_type", SUPPORTED_MAPPING_TYPES)
-def test_get_log_data_as_dict_with_mapping_types(
- mapping_data: Mapping, mapping_type: Callable
-) -> None:
- log_data = mapping_type(mapping_data)
- data_dict = core._get_log_data_as_dict(log_data)
- assert len(data_dict) == len(mapping_data)
- assert set(data_dict.keys()) == set(map(str, mapping_data.keys()))
- assert set(data_dict.values()) == set(mapping_data.values())
- # Dictionary returned should be invariant to original ordering
- assert data_dict == core._get_log_data_as_dict(
- mapping_type(dict(reversed(mapping_data.items())))
- )
-
-
-@pytest.mark.parametrize("mapping_data", MAPPING_DATA_VALUES)
-def test_get_log_data_as_dict_with_multirow_dataframe_raises(
- mapping_data: Mapping,
-) -> None:
- log_data = pd.DataFrame(mapping_data, index=[0, 1])
- with pytest.raises(ValueError, match="multirow"):
- core._get_log_data_as_dict(log_data)
-
-
-@pytest.mark.parametrize("values", ITERABLE_DATA_VALUES)
-@pytest.mark.parametrize("sequence_type", SUPPORTED_SEQUENCE_TYPES)
-def test_get_log_data_as_dict_with_sequence_types(
- values: Iterable, sequence_type: Callable
-) -> None:
- log_data = sequence_type(values)
- data_dict = core._get_log_data_as_dict(log_data)
- assert len(data_dict) == len(log_data)
- assert list(data_dict.keys()) == [f"item_{i+1}" for i in range(len(log_data))]
- assert list(data_dict.values()) == list(log_data)
-
-
-@pytest.mark.parametrize("values", ITERABLE_DATA_VALUES)
-def test_get_log_data_as_dict_with_set(values: Iterable) -> None:
- data = set(values)
- data_dict = core._get_log_data_as_dict(data)
- assert len(data_dict) == len(data)
- assert list(data_dict.keys()) == [f"item_{i+1}" for i in range(len(data))]
- assert set(data_dict.values()) == data
- # Dictionary returned should be invariant to original ordering
- assert data_dict == core._get_log_data_as_dict(set(reversed(values)))
-
-
-def test_convert_numpy_scalars_to_python_types() -> None:
- data = {
- "a": np.int64(1),
- "b": np.int32(42),
- "c": np.float64(0.5),
- "d": np.bool_(True),
- }
- expected_converted_data = {"a": 1, "b": 42, "c": 0.5, "d": True}
- converted_data = core._convert_numpy_scalars_to_python_types(data)
- assert converted_data == expected_converted_data
-
-
-def test_get_columns_from_data_dict() -> None:
- data = {
- "a": 1,
- "b": 0.5,
- "c": False,
- "d": "foo",
- "e": pd.Timestamp("2010-01-01"),
- }
- expected_columns = {
- "a": "int",
- "b": "float",
- "c": "bool",
- "d": "str",
- "e": "Timestamp",
- }
- columns = core._get_columns_from_data_dict(data)
- assert columns == expected_columns
-
-
-@contextlib.contextmanager
-def _propagate_to_root() -> Generator[None, None, None]:
- # Enable propagation to root logger to allow pytest capturing to work
- root_logger = logging.getLogger("tlo")
- root_logger._std_logger.propagate = True
- yield
- root_logger._std_logger.propagate = False
-
-
-def _setup_caplog_and_get_logger(
- caplog: pytest.LogCaptureFixture, logger_name: str, logger_level: core.LogLevel
-) -> core.Logger:
- caplog.set_level(CATCH_ALL_LEVEL, logger_name)
+from tlo import Date, Simulation, logging
+from tlo.methods import demography, enhanced_lifestyle
+
+start_date = Date(2010, 1, 1)
+popsize = 500
+
+
+@pytest.fixture(scope='function')
+def basic_configuration(tmpdir):
+ """Setup basic file handler configuration"""
+ # tlo module config
+ file_name = tmpdir.join('test.log')
+ file_handler = logging.set_output_file(file_name)
+
+ yield file_handler, file_name
+
+ file_handler.close()
+
+
+@pytest.fixture(scope='function')
+def simulation_configuration(tmpdir):
+ resourcefilepath = Path(os.path.dirname(__file__)) / '../resources'
+
+ sim = Simulation(start_date=start_date, log_config={'filename': 'log', 'directory': tmpdir})
+ sim.register(demography.Demography(resourcefilepath=resourcefilepath),
+ enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath))
+
+ yield sim.output_file, sim.log_filepath
+
+ sim.output_file.close()
+
+
+def read_file(file_handler, file_name):
+ """
+ Reads file and returns the lines
+ :param file_handler: filehandler (to flush) though might be a bit unnecessary
+ :param file_name: path to file
+ :return: list of lines
+ """
+ file_handler.flush()
+ with open(file_name) as handle:
+ lines = handle.readlines()
+ return lines
+
+
+def log_message(message_level, logger_level, message, logger_name='tlo.test.logger', structured_logging=False):
+ """
+ Sets up logger level, and writes message at the message level
+
+ :param message_level: level that the message will be added as
+ :param logger_level: level that the logger is set to
+ :param message: message to be written to log
+ :param structured_logging:
+
+ """
logger = logging.getLogger(logger_name)
logger.setLevel(logger_level)
- return logger
-
-
-@pytest.mark.parametrize("disable_level", LOGGING_LEVELS, ids=_logging.getLevelName)
-@pytest.mark.parametrize("logger_level_offset", [-5, 0, 5])
-@pytest.mark.parametrize("data", STRING_DATA_VALUES)
-@pytest.mark.parametrize("logger_name", LOGGER_NAMES)
-def test_disable(
- disable_level: core.LogLevel,
- logger_level_offset: int,
- data: str,
- logger_name: str,
- caplog: pytest.LogCaptureFixture,
-) -> None:
- logger = _setup_caplog_and_get_logger(caplog, logger_name, CATCH_ALL_LEVEL)
- logging.disable(disable_level)
- assert not logger.isEnabledFor(disable_level)
- message_level = disable_level + logger_level_offset
- with _propagate_to_root():
- logger.log(message_level, key="message", data=data)
- if message_level > disable_level:
- # Message level is above disable level and so should have been captured
- assert len(caplog.records) == 1
- assert data in caplog.records[0].msg
- else:
- # Message level is below disable level and so should not have been captured
- assert len(caplog.records) == 0
-
-
-def _check_captured_log_output_for_levels(
- caplog: pytest.LogCaptureFixture,
- message_level: core.LogLevel,
- logger_level: core.LogLevel,
- data: str,
-) -> None:
- if message_level >= logger_level:
- # Message level is at or above logger's level and so should have been captured
- assert len(caplog.records) == 1
- assert data in caplog.records[0].msg
+
+ if structured_logging:
+ if message_level == 'logging.DEBUG':
+ logger.debug(key='structured', data=message)
+ elif message_level == 'logging.INFO':
+ logger.info(key='structure', data=message)
+ elif message_level == 'logging.WARNING':
+ logger.warning(key='structured', data=message)
+ elif message_level == 'logging.CRITICAL':
+ logger.critical(key='structured', data=message)
else:
- # Message level is below logger's set level and so should not have been captured
- assert len(caplog.records) == 0
-
-
-@pytest.mark.parametrize("message_level", LOGGING_LEVELS, ids=_logging.getLevelName)
-@pytest.mark.parametrize("logger_level_offset", [-5, 0, 5])
-@pytest.mark.parametrize("data", STRING_DATA_VALUES)
-@pytest.mark.parametrize("logger_name", LOGGER_NAMES)
-def test_logging_with_log(
- message_level: core.LogLevel,
- logger_level_offset: int,
- data: str,
- logger_name: str,
- caplog: pytest.LogCaptureFixture,
-) -> None:
- logger_level = message_level + logger_level_offset
- logger = _setup_caplog_and_get_logger(caplog, logger_name, logger_level)
- with _propagate_to_root():
- logger.log(level=message_level, key="message", data=data)
- _check_captured_log_output_for_levels(caplog, message_level, logger_level, data)
-
-
-@pytest.mark.parametrize("message_level", LOGGING_LEVELS, ids=_logging.getLevelName)
-@pytest.mark.parametrize("logger_level_offset", [-5, 0, 5])
-@pytest.mark.parametrize("logger_name", LOGGER_NAMES)
-@pytest.mark.parametrize("data", STRING_DATA_VALUES)
-def test_logging_with_convenience_methods(
- message_level: core.LogLevel,
- logger_level_offset: int,
- data: str,
- logger_name: str,
- caplog: pytest.LogCaptureFixture,
-) -> None:
- logger_level = message_level + logger_level_offset
- logger = _setup_caplog_and_get_logger(caplog, logger_name, logger_level)
- convenience_method = getattr(logger, _logging.getLevelName(message_level).lower())
- with _propagate_to_root():
- convenience_method(key="message", data=data)
- _check_captured_log_output_for_levels(caplog, message_level, logger_level, data)
-
-
-def _check_header(
- header: dict[str, str | dict[str, str]],
- expected_module: str,
- expected_key: str,
- expected_level: str,
- expected_description: str,
- expected_columns: dict[str, str],
-) -> None:
- assert set(header.keys()) == {
- "uuid",
- "type",
- "module",
- "key",
- "level",
- "columns",
- "description",
- }
- assert isinstance(header["uuid"], str)
- assert set(header["uuid"]) <= set("abcdef0123456789")
- assert header["type"] == "header"
- assert header["module"] == expected_module
- assert header["key"] == expected_key
- assert header["level"] == expected_level
- assert header["description"] == expected_description
- assert isinstance(header["columns"], dict)
- assert header["columns"] == expected_columns
-
-
-def _check_row(
- row: dict[str, str],
- logger_level: core.LogLevel,
- expected_uuid: str,
- expected_date: str,
- expected_values: list,
- expected_module: str,
- expected_key: str,
-) -> None:
- assert row["uuid"] == expected_uuid
- assert row["date"] == expected_date
- assert row["values"] == expected_values
- if logger_level == logging.DEBUG:
- assert row["module"] == expected_module
- assert row["key"] == expected_key
-
-
-def _parse_and_check_log_records(
- caplog: pytest.LogCaptureFixture,
- logger_name: str,
- logger_level: core.LogLevel,
- message_level: core.LogLevel,
- data_dicts: dict,
- dates: str,
- keys: str,
- description: str | None = None,
-) -> None:
- headers = {}
- for record, data_dict, date, key in zip(caplog.records, data_dicts, dates, keys):
- message_lines = record.msg.split("\n")
- if key not in headers:
- # First record for key therefore expect both header and row lines
- assert len(message_lines) == 2
- header_line, row_line = message_lines
- headers[key] = json.loads(header_line)
- _check_header(
- header=headers[key],
- expected_module=logger_name,
- expected_key=key,
- expected_level=_logging.getLevelName(logger_level),
- expected_description=description,
- expected_columns=logging.core._get_columns_from_data_dict(data_dict),
- )
- else:
- # Subsequent records for key should only have row line
- assert len(message_lines) == 1
- row_line = message_lines[0]
- row = json.loads(row_line)
- _check_row(
- row=row,
- logger_level=message_level,
- expected_uuid=headers[key]["uuid"],
- expected_date=date,
- expected_values=list(data_dict.values()),
- expected_module=logger_name,
- expected_key=key,
- )
-
-
-@pytest.mark.parametrize("level", LOGGING_LEVELS, ids=_logging.getLevelName)
-@pytest.mark.parametrize(
- "data_type,data",
- list(
- chain(
- zip([str] * len(STRING_DATA_VALUES), STRING_DATA_VALUES),
- product(SUPPORTED_ITERABLE_TYPES, ITERABLE_DATA_VALUES),
- product(SUPPORTED_MAPPING_TYPES, MAPPING_DATA_VALUES),
- )
- ),
-)
-@pytest.mark.parametrize("logger_name", LOGGER_NAMES)
-@pytest.mark.parametrize("key", STRING_DATA_VALUES)
-@pytest.mark.parametrize("description", [None, "test"])
-@pytest.mark.parametrize("number_repeats", [1, 2, 3])
-def test_logging_structured_data(
- level: core.LogLevel,
- data_type: Callable,
- data: Mapping | Iterable,
- logger_name: str,
- key: str,
- description: str,
- number_repeats: int,
- caplog: pytest.LogCaptureFixture,
-) -> None:
- logger = _setup_caplog_and_get_logger(caplog, logger_name, level)
- log_data = data_type(data)
- data_dict = logging.core._get_log_data_as_dict(log_data)
- with _propagate_to_root():
- for _ in range(number_repeats):
- logger.log(level=level, key=key, data=log_data, description=description)
- assert len(caplog.records) == number_repeats
- _parse_and_check_log_records(
- caplog=caplog,
- logger_name=logger_name,
- logger_level=level,
- message_level=level,
- data_dicts=repeat(data_dict),
- dates=repeat(SIMULATION_DATE),
- keys=repeat(key),
- description=description,
- )
-
-
-@pytest.mark.parametrize("simulation_date_getter", [UpdateableSimulateDateGetter()])
-@pytest.mark.parametrize("logger_name", LOGGER_NAMES)
-@pytest.mark.parametrize("number_dates", [2, 3])
-def test_logging_updating_simulation_date(
- simulation_date_getter: core.SimulationDateGetter,
- logger_name: str,
- root_level: core.LogLevel,
- number_dates: int,
- caplog: pytest.LogCaptureFixture,
-) -> None:
- logger = _setup_caplog_and_get_logger(caplog, logger_name, root_level)
- key = "message"
- data = "spam"
- data_dict = logging.core._get_log_data_as_dict(data)
- dates = []
- with _propagate_to_root():
- for _ in range(number_dates):
- logger.log(level=root_level, key=key, data=data)
- dates.append(simulation_date_getter())
- simulation_date_getter.increment_date()
- # Dates should be unique
- assert len(set(dates)) == len(dates)
- assert len(caplog.records) == number_dates
- _parse_and_check_log_records(
- caplog=caplog,
- logger_name=logger_name,
- logger_level=root_level,
- message_level=root_level,
- data_dicts=repeat(data_dict),
- dates=dates,
- keys=repeat(key),
- description=None,
- )
-
-
-@pytest.mark.parametrize("logger_name", LOGGER_NAMES)
-def test_logging_structured_data_multiple_keys(
- logger_name: str,
- root_level: core.LogLevel,
- caplog: pytest.LogCaptureFixture,
-) -> None:
- logger = _setup_caplog_and_get_logger(caplog, logger_name, root_level)
- keys = ["foo", "bar", "foo", "foo", "bar"]
- data_values = ["a", "b", "c", "d", "e"]
- data_dicts = [logging.core._get_log_data_as_dict(data) for data in data_values]
- with _propagate_to_root():
- for key, data in zip(keys, data_values):
- logger.log(level=root_level, key=key, data=data)
- assert len(caplog.records) == len(keys)
- _parse_and_check_log_records(
- caplog=caplog,
- logger_name=logger_name,
- logger_level=root_level,
- message_level=root_level,
- data_dicts=data_dicts,
- dates=repeat(SIMULATION_DATE),
- keys=keys,
- description=None,
- )
-
-
-@pytest.mark.parametrize("level", LOGGING_LEVELS)
-def test_logging_to_file(level: core.LogLevel, tmp_path: Path) -> None:
- log_path = tmp_path / "test.log"
- file_handler = logging.set_output_file(log_path)
- loggers = [logging.getLogger(name) for name in LOGGER_NAMES]
- key = "message"
- for logger, data in zip(loggers, STRING_DATA_VALUES):
- logger.setLevel(level)
- logger.log(level=level, key=key, data=data)
- _logging.shutdown([lambda: file_handler])
- with log_path.open("r") as log_file:
- log_lines = log_file.readlines()
- # Should have two lines (one header + one data row per logger)
- assert len(log_lines) == 2 * len(loggers)
- for name, data in zip(LOGGER_NAMES, STRING_DATA_VALUES):
- header = json.loads(log_lines.pop(0))
- row = json.loads(log_lines.pop(0))
- _check_header(
- header=header,
- expected_module=name,
- expected_key=key,
- expected_level=_logging.getLevelName(level),
- expected_description=None,
- expected_columns={key: "str"},
- )
- _check_row(
- row=row,
- logger_level=level,
- expected_uuid=header["uuid"],
- expected_date=SIMULATION_DATE,
- expected_values=[data],
- expected_module=name,
- expected_key=key,
- )
-
-
-@pytest.mark.parametrize(
- "inconsistent_data_iterables",
- [
- ({"a": 1, "b": 2}, {"a": 3, "b": 4, "c": 5}),
- ({"a": 1}, {"b": 2}),
- ({"a": None, "b": 2}, {"a": 1, "b": 2}),
- ([1], [0.5]),
- (["a", "b"], ["a", "b", "c"]),
- ("foo", "bar", ["spam"]),
- ],
-)
-def test_logging_structured_data_inconsistent_columns_warns(
- inconsistent_data_iterables: Iterable[core.LogData], root_level: core.LogLevel
-) -> None:
- logger = logging.getLogger("tlo")
- with pytest.warns(core.InconsistentLoggedColumnsWarning):
- for data in inconsistent_data_iterables:
- logger.log(level=root_level, key="message", data=data)
-
-
-@pytest.mark.parametrize(
- "consistent_data_iterables",
- [
- ([np.int64(1)], [2], [np.int32(1)]),
- ([{"a": np.bool_(False)}, {"a": False}]),
- ((1.5, 2), (np.float64(0), np.int64(2))),
- ],
-)
-@pytest.mark.filterwarnings("error")
-def test_logging_structured_data_mixed_numpy_python_scalars(
- consistent_data_iterables: Iterable[core.LogData], root_level: core.LogLevel
-) -> None:
- logger = logging.getLogger("tlo")
- # Should run without any exceptions
- for data in consistent_data_iterables:
- logger.log(level=root_level, key="message", data=data)
+ if message_level == 'logging.DEBUG':
+ logger.debug(message)
+ elif message_level == 'logging.INFO':
+ logger.info(message)
+ elif message_level == 'logging.WARNING':
+ logger.warning(message)
+ elif message_level == 'logging.CRITICAL':
+ logger.critical(message)
+
+
+class TestStructuredLogging:
+ @pytest.mark.parametrize("message_level", ["logging.DEBUG", "logging.INFO", "logging.WARNING", "logging.CRITICAL"])
+ def test_messages_same_level(self, simulation_configuration, message_level):
+ # given that messages are at the same level as the logger
+ logger_level = eval(message_level)
+ message = {"message": pd.Series([12.5])[0]}
+ file_handler, file_path = simulation_configuration
+ log_message(message_level, logger_level, message, structured_logging=True)
+
+ lines = read_file(file_handler, file_path)
+ header_json = json.loads(lines[5])
+ data_json = json.loads(lines[6])
+
+ # message should be written to log
+ assert len(lines) == 7
+ assert header_json['level'] == message_level.lstrip("logging.")
+ assert 'message' in header_json['columns']
+ assert header_json['columns']['message'] == 'float64'
+ assert data_json['values'] == [12.5]
+
+ @pytest.mark.parametrize("message_level", ["logging.DEBUG", "logging.INFO", "logging.WARNING", "logging.CRITICAL"])
+ def test_messages_higher_level(self, simulation_configuration, message_level):
+ # given that messages are a higher level than the logger
+ logger_level = eval(message_level) - 1
+ message = {"message": pd.Series([12.5])[0]}
+ file_handler, file_path = simulation_configuration
+ log_message(message_level, logger_level, message, structured_logging=True)
+
+ lines = read_file(file_handler, file_path)
+ header_json = json.loads(lines[5])
+ data_json = json.loads(lines[6])
+
+ # message should be written to log
+ assert len(lines) == 7
+ assert header_json['level'] == message_level.lstrip("logging.")
+ assert 'message' in header_json['columns']
+ assert header_json['columns']['message'] == 'float64'
+ assert data_json['values'] == [12.5]
+
+ @pytest.mark.parametrize("message_level", ["logging.DEBUG", "logging.INFO", "logging.WARNING", "logging.CRITICAL"])
+ def test_messages_lower_level(self, simulation_configuration, message_level):
+ # given that messages are at a lower level than logger
+ logger_level = eval(message_level) + 1
+ message = {"message": pd.Series([12.5])[0]}
+ file_handler, file_path = simulation_configuration
+ log_message(message_level, logger_level, message, structured_logging=True)
+
+ lines = read_file(file_handler, file_path)
+
+ # only simulation info messages should be written to log
+ assert len(lines) == 5
+
+
+class TestConvertLogData:
+ def setup_method(self):
+ self.expected_output = {'item_1': 1, 'item_2': 2}
+ self.logger = logging.getLogger('tlo.test.logger')
+
+ @pytest.mark.parametrize("iterable_data", [[1, 2], {1, 2}, (1, 2)])
+ def test_convert_iterable_to_dict(self, iterable_data):
+ output = self.logger._get_data_as_dict(iterable_data)
+ assert self.expected_output == output
+
+ def test_convert_df_to_dict(self):
+ df = pd.DataFrame({'item_1': [1], 'item_2': [2]})
+ output = self.logger._get_data_as_dict(df)
+
+ assert self.expected_output == output
+
+ def test_string_to_dict(self):
+ output = self.logger._get_data_as_dict("strings")
+ assert {'message': 'strings'} == output
+
+
+def test_mixed_logging():
+ """Logging with both oldstyle and structured logging should raise an error"""
+ logger = logging.getLogger('tlo.test.logger')
+ logger.setLevel(logging.INFO)
+ with pytest.raises(ValueError):
+ logger.info("stdlib method")
+ logger.info(key="structured", data={"key": 10})
+
+
+@pytest.mark.parametrize("add_stdout_handler", ((True, False)))
+def test_init_logging(add_stdout_handler):
+ logging.init_logging(add_stdout_handler)
+ logger = logging.getLogger('tlo')
+ assert len(logger.handlers) == (1 if add_stdout_handler else 0)
diff --git a/tests/test_logging_end_to_end.py b/tests/test_logging_end_to_end.py
index 944c3021c4..5f055c95ab 100644
--- a/tests/test_logging_end_to_end.py
+++ b/tests/test_logging_end_to_end.py
@@ -16,13 +16,13 @@ def log_input():
log_string = "\n".join((
"col1_str;hello;world;lorem;ipsum;dolor;sit",
"col2_int;1;3;5;7;8;10",
- "col3_float;2.1;4.1;6.1;8.1;9.1;0.1",
+ "col3_float;2;4;6;8;9;null",
"col4_cat;cat1;cat1;cat2;cat2;cat1;cat2",
- "col5_set;{'zero'};{'one'};{'two'};{'three'};{'four'};{'five'}",
- "col6_list;[1, 3];[2, 4];[0, 3];[5, 6];[7, 8];[9, 10]",
+ "col5_set;set();{'one'};{None};{'three','four'};{'eight'};set()",
+ "col6_list;[];['two'];[None];[5, 6, 7];[];[]",
"col7_date;2020-06-19T00:22:58.586101;2020-06-20T00:23:58.586101;2020-06-21T00:24:58.586101;2020-06-22T00:25"
- ":58.586101;2020-06-23T00:25:58.586101;2020-06-21T00:24:58.586101",
- "col8_fixed_list;['one', 1];['two', 2];['three', 3];['three', 3];['four', 4];['five', 5]"
+ ":58.586101;2020-06-23T00:25:58.586101;null",
+ "col8_fixed_list;['one', 1];['two', 2];[None, None];['three', 3];['four', 4];['five', 5]"
))
# read in, then transpose
log_input = pd.read_csv(StringIO(log_string), sep=';').T
@@ -63,6 +63,8 @@ def log_path(tmpdir_factory, log_input, class_scoped_seed):
# a logger connected to that simulation
logger = logging.getLogger('tlo.test')
logger.setLevel(logging.INFO)
+ # Allowing logging of entire dataframe only for testing
+ logger._disable_dataframe_logging = False
# log data as dicts
for index, row in log_input.iterrows():
@@ -74,9 +76,15 @@ def log_path(tmpdir_factory, log_input, class_scoped_seed):
logger.info(key='rows_as_individuals', data=log_input.loc[[index]])
sim.date = sim.date + pd.DateOffset(days=1)
+ # log data as multi-row dataframe
+ for _ in range(2):
+ logger.info(key='multi_row_df', data=log_input)
+ sim.date = sim.date + pd.DateOffset(days=1)
+
# log data as fixed length list
for item in log_input.col8_fixed_list.values:
- logger.info(key='a_fixed_length_list', data=item)
+ logger.info(key='a_fixed_length_list',
+ data=item)
sim.date = sim.date + pd.DateOffset(days=1)
# log data as variable length list
@@ -129,12 +137,26 @@ def test_rows_as_individuals(self, test_log_df, log_input):
log_output.col4_cat = log_output.col4_cat.astype('category')
assert log_input.equals(log_output)
+ def test_log_entire_df(self, test_log_df, log_input):
+ # get table to compare
+ log_output = test_log_df['multi_row_df'].drop(['date'], axis=1)
+
+ # within nested dicts/entire df, need manual setting of special types
+ log_output.col4_cat = log_output.col4_cat.astype('category')
+ log_input.col5_set = log_input.col5_set.apply(list)
+ log_output.col7_date = log_output.col7_date.astype('datetime64[ns]')
+ # deal with index matching by resetting index
+ log_output.reset_index(inplace=True, drop=True)
+ expected_output = pd.concat((log_input, log_input), ignore_index=True)
+
+ assert expected_output.equals(log_output)
+
def test_fixed_length_list(self, test_log_df):
log_df = test_log_df['a_fixed_length_list'].drop(['date'], axis=1)
expected_output = pd.DataFrame(
- {'item_1': ['one', 'two', 'three', 'three', 'four', 'five'],
- 'item_2': [1, 2, 3, 3, 4, 5]}
+ {'item_1': ['one', 'two', None, 'three', 'four', 'five'],
+ 'item_2': [1, 2, None, 3, 4, 5]}
)
assert expected_output.equals(log_df)
diff --git a/tests/test_malaria.py b/tests/test_malaria.py
index 2b16da0000..4ac2d377db 100644
--- a/tests/test_malaria.py
+++ b/tests/test_malaria.py
@@ -268,7 +268,7 @@ def test_dx_algorithm_for_malaria_outcomes_clinical(
add_or_remove='+'
)
- assert "fever" in sim.modules["SymptomManager"].has_what(person_id=person_id)
+ assert "fever" in sim.modules["SymptomManager"].has_what(person_id)
def diagnosis_function(tests, use_dict: bool = False, report_tried: bool = False):
return hsi_event.healthcare_system.dx_manager.run_dx_test(
@@ -281,7 +281,7 @@ def diagnosis_function(tests, use_dict: bool = False, report_tried: bool = False
assert sim.modules['Malaria'].check_if_fever_is_caused_by_malaria(
true_malaria_infection_type = df.at[person_id, "ma_inf_type"],
diagnosis_function = diagnosis_function,
- person_id=person_id,
+ patient_id=person_id,
) == expected_diagnosis
@@ -346,7 +346,7 @@ def make_blank_simulation():
add_or_remove='+'
)
- assert "fever" in sim.modules["SymptomManager"].has_what(person_id=person_id)
+ assert "fever" in sim.modules["SymptomManager"].has_what(person_id)
def diagnosis_function(tests, use_dict: bool = False, report_tried: bool = False):
return hsi_event.healthcare_system.dx_manager.run_dx_test(
@@ -362,7 +362,7 @@ def diagnosis_function(tests, use_dict: bool = False, report_tried: bool = False
person_id, "ma_inf_type"
],
diagnosis_function=diagnosis_function,
- person_id=person_id,
+ patient_id=person_id,
)
== "negative_malaria_test"
)
@@ -517,7 +517,7 @@ def test_individual_testing_and_treatment(sim):
pollevent.run()
assert not pd.isnull(df.at[person_id, "ma_date_symptoms"])
- assert set(sim.modules['SymptomManager'].has_what(person_id=person_id)) == {"fever", "headache", "vomiting", "stomachache"}
+ assert set(sim.modules['SymptomManager'].has_what(person_id)) == {"fever", "headache", "vomiting", "stomachache"}
# check rdt is scheduled
date_event, event = [
@@ -560,7 +560,7 @@ def test_individual_testing_and_treatment(sim):
pollevent = malaria.MalariaUpdateEvent(module=sim.modules['Malaria'])
pollevent.apply(sim.population)
- assert sim.modules['SymptomManager'].has_what(person_id=person_id) == []
+ assert sim.modules['SymptomManager'].has_what(person_id) == []
# check no rdt is scheduled
assert "malaria.HSI_Malaria_rdt" not in sim.modules['HealthSystem'].find_events_for_person(person_id)
diff --git a/tests/test_maternal_health_helper_and_analysis_functions.py b/tests/test_maternal_health_helper_and_analysis_functions.py
index daea95a5e6..83a93c224d 100644
--- a/tests/test_maternal_health_helper_and_analysis_functions.py
+++ b/tests/test_maternal_health_helper_and_analysis_functions.py
@@ -258,9 +258,9 @@ def test_analysis_events_force_availability_of_consumables_when_scheduled_in_anc
syph_test = module.item_codes_preg_consumables['syphilis_test']
syph_treat = module.item_codes_preg_consumables['syphilis_treatment']
- for cons in 'iron_folic_acid', 'balanced_energy_protein', 'calcium', 'syphilis_test', 'syphilis_treatment':
- updated_cons = {k: v * 0 for (k, v) in module.item_codes_preg_consumables[cons].items()}
- sim.modules['HealthSystem'].override_availability_of_consumables(updated_cons)
+ for cons in iron, protein, calcium, syph_test, syph_treat:
+ sim.modules['HealthSystem'].override_availability_of_consumables(
+ {cons[0]: 0.0})
# refresh the consumables
sim.modules['HealthSystem'].consumables._refresh_availability_of_consumables(date=sim.date)
diff --git a/tests/test_module_dependencies.py b/tests/test_module_dependencies.py
index 8ed5b6811e..fd61bb40be 100644
--- a/tests/test_module_dependencies.py
+++ b/tests/test_module_dependencies.py
@@ -1,4 +1,5 @@
"""Tests for automatic checking and ordering of method module dependencies."""
+
import os
from pathlib import Path
from random import seed as set_seed
@@ -7,7 +8,7 @@
import pytest
-from tlo import Date, Module, Simulation, logging
+from tlo import Date, Module, Simulation
from tlo.dependencies import (
ModuleDependencyError,
get_all_dependencies,
@@ -16,7 +17,6 @@
get_module_class_map,
topologically_sort_modules,
)
-from tlo.methods import hiv, simplified_births
try:
resourcefilepath = Path(os.path.dirname(__file__)) / "../resources"
@@ -28,12 +28,9 @@
simulation_end_date = Date(2010, 9, 1)
simulation_initial_population = 1000
+
module_class_map = get_module_class_map(
- excluded_modules={
- "Module",
- "Skeleton",
- "SimplifiedPregnancyAndLabour",
- }
+ excluded_modules={'Module', 'Skeleton', 'SimplifiedPregnancyAndLabour'}
)
@@ -50,6 +47,7 @@ def sim(seed):
@pytest.fixture
def dependent_module_pair():
+
class Module1(Module):
pass
@@ -65,7 +63,7 @@ def dependent_module_chain():
type(
f'Module{i}',
(Module,),
- {'INIT_DEPENDENCIES': frozenset({f'Module{i - 1}'})} if i != 0 else {}
+ {'INIT_DEPENDENCIES': frozenset({f'Module{i-1}'})} if i != 0 else {}
)
for i in range(10)
]
@@ -249,8 +247,8 @@ def test_module_dependencies_complete(sim, module_class):
for module in module_class_map.values()
# Skip test for NewbornOutcomes as long simulation needed for birth events to occur and dependencies to be used
if module.__name__ not in {
- 'NewbornOutcomes'
- }
+ 'NewbornOutcomes'
+ }
for dependency_name in sorted(get_all_required_dependencies(module))
],
ids=lambda pair: f"{pair[0].__name__}, {pair[1].__name__}"
@@ -283,76 +281,3 @@ def test_module_dependencies_all_required(sim, module_and_dependency_pair):
'does not appear to be required to run simulation without errors and so '
f'should be removed from the dependencies of {module_class.__name__}.'
)
-
-
-def test_auto_register_module_dependencies(tmpdir):
- """ check if module dependencies are registered as expected when an argument to auto register modules in simulation
- is set to True """
- # configure logging
- log_config = {
- 'filename': 'LogFile',
- 'directory': tmpdir,
- 'custom_levels': {
- '*': logging.CRITICAL,
- 'tlo.method.demography': logging.INFO
- }
- }
- # set simulation start date
- start_date = Date(2010, 1, 1)
-
- # register required modules for a simple simulation. We have included copd for as it has some dependencies. We want
- # to test if the dependencies can be automatically registered when the auto register argument in simulation
- # is set to True
- def register_disease_modules_manually():
- """ Test manually registering disease modules without including all dependencies and leaving to false an
- option to auto register missing dependencies. This should fail with module dependency error """
- with pytest.raises(ModuleDependencyError, match='missing'):
- # configure simulation
- sim = Simulation(start_date=start_date, seed=0, log_config=log_config, resourcefilepath=resourcefilepath)
- # the lines below should fail with missing dependencies
- sim.register(hiv.Hiv(resourcefilepath=resourcefilepath))
-
- def register_disease_modules_using_labour_modules_for_births():
- """ Test registering disease modules without including all dependencies and not using simplified births
- module BUT setting to true an option to auto register missing dependencies. This should register all necessary
- modules including all labour modules """
- # configure simulation
- sim = Simulation(start_date=start_date, seed=0, log_config=log_config, resourcefilepath=resourcefilepath)
- # re-register modules with auto-register-module argument set to True and using labour modules for births
- sim.register(hiv.Hiv(resourcefilepath=resourcefilepath),
- auto_register_dependencies=True)
- # get module dependencies
- required_dependencies = get_all_required_dependencies(sim.modules["Hiv"])
- # check registered dependencies
- registered_module_names = set(sim.modules.keys())
- # all required dependencies should be available in registered dependencies
- assert required_dependencies <= registered_module_names
-
- def register_disease_modules_using_simplified_births_for_births():
- """ Test registering disease modules without including all dependencies BUT setting to true an option to auto
- register missing dependencies and using simplified births module.This should register all necessary modules
- except labour modules since we're using simplified births """
- # configure simulation
- sim = Simulation(start_date=start_date, seed=0, log_config=log_config, resourcefilepath=resourcefilepath)
- sim.register(hiv.Hiv(resourcefilepath=resourcefilepath),
- simplified_births.SimplifiedBirths(resourcefilepath=resourcefilepath),
- auto_register_dependencies=True
- )
- # now that we're using simplified births we want to ensure that all alternative dependencies are not registered
- alternative_dependencies = simplified_births.SimplifiedBirths.ALTERNATIVE_TO
- # get registered modules
- registered_module_names = set(sim.modules.keys())
- # no alternative dependency(labour modules) should get registered when using simplified births
- for dependency in alternative_dependencies:
- assert dependency not in registered_module_names, (f'{dependency} should not be registered when simplified'
- f' module has been registered')
-
- # test registering disease modules manually(when all dependencies are not included and auto register missing
- # dependencies option is set to false)
- register_disease_modules_manually()
-
- # test auto registering disease modules using labor modules for births
- register_disease_modules_using_labour_modules_for_births()
-
- # test auto registering disease modules using simplified module for births
- register_disease_modules_using_simplified_births_for_births()
diff --git a/tests/test_population.py b/tests/test_population.py
deleted file mode 100644
index e8a549209d..0000000000
--- a/tests/test_population.py
+++ /dev/null
@@ -1,216 +0,0 @@
-import numpy as np
-import pandas as pd
-import pytest
-
-from tlo.core import Property, Types
-from tlo.population import Population
-
-
-@pytest.fixture
-def properties():
- return {
- f"{type_.name.lower()}_{i}": Property(type_, f"Column {i} of type {type_}")
- for type_ in [Types.INT, Types.BOOL, Types.REAL, Types.DATE, Types.BITSET]
- for i in range(5)
- }
-
-
-@pytest.fixture(params=[1, 100, 1000])
-def initial_size(request):
- return request.param
-
-
-@pytest.fixture(params=[None, 0.02, 0.1])
-def append_size(request, initial_size):
- return (
- request.param
- if request.param is None
- else max(int(initial_size * request.param), 1)
- )
-
-
-@pytest.fixture
-def population(properties, initial_size, append_size):
- return Population(properties, initial_size, append_size)
-
-
-@pytest.fixture
-def rng(seed):
- return np.random.RandomState(seed % 2**32)
-
-
-def _generate_random_values(property, rng, size=None):
- if property.type_ == Types.DATE:
- return np.datetime64("2010-01-01") + rng.randint(0, 4000, size=size)
- elif property.type_ in (Types.INT, Types.BITSET):
- return rng.randint(low=0, high=100, size=size)
- elif property.type_ == Types.REAL:
- return rng.standard_normal(size=size)
- elif property.type_ == Types.BOOL:
- return rng.uniform(size=size) < 0.5
- else:
- msg = f"Unhandled type {property.type_}"
- raise ValueError(msg)
-
-
-@pytest.fixture
-def population_with_random_property_values(population, properties, initial_size, rng):
-
- for name, property in properties.items():
- population.props[name] = pd.Series(
- _generate_random_values(property, rng, initial_size),
- dtype=property.pandas_type,
- )
-
- return population
-
-
-def test_population_invalid_append_size_raises(properties, initial_size):
- with pytest.raises(AssertionError, match="greater than 0"):
- Population(properties, initial_size, append_size=-1)
-
-
-def test_population_attributes(population, properties, initial_size, append_size):
- assert population.initial_size == initial_size
- assert population.next_person_id == initial_size
- if append_size is not None:
- assert len(population.new_rows) == append_size
- else:
- assert 0 < len(population.new_rows) <= initial_size
- assert len(population.props.index) == initial_size
- assert len(population.props.columns) == len(properties)
- assert set(population.props.columns) == properties.keys()
- assert all(
- properties[name].pandas_type == col.dtype
- for name, col in population.props.items()
- )
-
-
-def test_population_do_birth(population):
- initial_population_props_copy = population.props.copy()
- initial_size = population.initial_size
- append_size = len(population.new_rows)
-
- def check_population(population, birth_number):
- expected_next_person_id = initial_size + birth_number
- # population size should increase by append_size on first birth and after
- # every subsequent append_size births by a further append_size
- expected_size = (
- initial_size + ((birth_number - 1) // append_size + 1) * append_size
- )
- assert all(initial_population_props_copy.columns == population.props.columns)
- assert all(initial_population_props_copy.dtypes == population.props.dtypes)
- assert population.next_person_id == expected_next_person_id
- assert len(population.props.index) == expected_size
-
- for birth_number in range(1, append_size + 2):
- population.do_birth()
- check_population(population, birth_number)
-
-
-def test_population_individual_properties_read_only_write_raises(
- population, properties
-):
- with population.individual_properties(
- person_id=0, read_only=True
- ) as individual_properties:
- for property_name in properties:
- with pytest.raises(ValueError, match="read-only"):
- individual_properties[property_name] = 0
-
-
-@pytest.mark.parametrize("read_only", [True, False])
-@pytest.mark.parametrize("person_id", [0, 1, -1])
-def test_population_individual_properties_read(
- population_with_random_property_values, properties, rng, read_only, person_id
-):
- person_id = person_id % population_with_random_property_values.initial_size
- population_dataframe = population_with_random_property_values.props
- with population_with_random_property_values.individual_properties(
- person_id=person_id, read_only=read_only
- ) as individual_properties:
- for property_name in properties:
- assert (
- individual_properties[property_name]
- == population_dataframe.at[person_id, property_name]
- )
- # Try reading all properties (in a new random order) a second time to check any
- # caching mechanism is working as expected
- shuffled_property_names = list(properties.keys())
- rng.shuffle(shuffled_property_names)
- for property_name in shuffled_property_names:
- assert (
- individual_properties[property_name]
- == population_dataframe.at[person_id, property_name]
- )
-
-
-@pytest.mark.parametrize("read_only", [True, False])
-@pytest.mark.parametrize("person_id", [0, 1, -1])
-def test_population_individual_properties_access_raises_when_finalized(
- population_with_random_property_values, properties, rng, read_only, person_id
-):
- person_id = person_id % population_with_random_property_values.initial_size
- with population_with_random_property_values.individual_properties(
- person_id=person_id, read_only=read_only
- ) as individual_properties:
- pass
- for property_name in properties:
- with pytest.raises(ValueError, match="finalized"):
- individual_properties[property_name]
- with pytest.raises(ValueError, match="finalized"):
- individual_properties[property_name] = None
-
-
-@pytest.mark.parametrize("person_id", [0, 1, -1])
-def test_population_individual_properties_write_with_context_manager(
- population_with_random_property_values, properties, rng, person_id
-):
- initial_population_dataframe = population_with_random_property_values.props.copy()
- person_id = person_id % population_with_random_property_values.initial_size
- updated_values = {}
- with population_with_random_property_values.individual_properties(
- person_id=person_id, read_only=False
- ) as individual_properties:
- for property_name, property in properties.items():
- updated_values[property_name] = _generate_random_values(property, rng)
- individual_properties[property_name] = updated_values[property_name]
- # Population dataframe should see updated properties for person_id row
- for property_name, property in properties.items():
- assert (
- population_with_random_property_values.props.at[person_id, property_name]
- == updated_values[property_name]
- )
- # All other rows in population dataframe should be unchanged
- all_rows_except_updated = ~initial_population_dataframe.index.isin([person_id])
- assert population_with_random_property_values.props[all_rows_except_updated].equals(
- initial_population_dataframe[all_rows_except_updated]
- )
-
-
-@pytest.mark.parametrize("person_id", [0, 1, -1])
-def test_population_individual_properties_write_with_sync(
- population_with_random_property_values, properties, rng, person_id
-):
- initial_population_dataframe = population_with_random_property_values.props.copy()
- person_id = person_id % population_with_random_property_values.initial_size
- updated_values = {}
- with population_with_random_property_values.individual_properties(
- person_id=person_id, read_only=False
- ) as individual_properties:
- for property_name, property in properties.items():
- updated_values[property_name] = _generate_random_values(property, rng)
- individual_properties[property_name] = updated_values[property_name]
- # Before synchronization all values in population dataframe should be unchanged
- assert initial_population_dataframe.equals(
- population_with_random_property_values.props
- )
- individual_properties.synchronize_updates_to_dataframe()
- # After synchronization all values in population dataframe should be updated
- for property_name, property in properties.items():
- assert (
- population_with_random_property_values.props.at[
- person_id, property_name
- ]
- == updated_values[property_name]
- )
diff --git a/tests/test_scenario.py b/tests/test_scenario.py
deleted file mode 100644
index 1feac8ef87..0000000000
--- a/tests/test_scenario.py
+++ /dev/null
@@ -1,68 +0,0 @@
-import json
-import os
-from pathlib import Path
-
-import pytest
-
-from tlo.scenario import BaseScenario, SampleRunner, ScenarioLoader
-
-
-@pytest.fixture
-def scenario_path():
- return Path(f'{os.path.dirname(__file__)}/resources/scenario.py')
-
-
-@pytest.fixture
-def pop_size():
- return 100
-
-
-@pytest.fixture
-def loaded_scenario(scenario_path):
- return ScenarioLoader(scenario_path).get_scenario()
-
-
-@pytest.fixture
-def arguments(pop_size):
- return ['--pop-size', str(pop_size)]
-
-
-@pytest.fixture
-def loaded_scenario_with_parsed_arguments(loaded_scenario, arguments):
- loaded_scenario.parse_arguments(arguments)
- return loaded_scenario
-
-
-def test_load(loaded_scenario, scenario_path):
- """Check we can load the scenario class from a file"""
- assert isinstance(loaded_scenario, BaseScenario)
- assert loaded_scenario.scenario_path == scenario_path
- assert hasattr(loaded_scenario, "pop_size") # Default value set in initialiser
-
-
-def test_parse_arguments(loaded_scenario_with_parsed_arguments, pop_size):
- """Check we can parse arguments related to the scenario. pop-size is used by our scenario,
- suspend-date is used in base class"""
- assert loaded_scenario_with_parsed_arguments.pop_size == pop_size
-
-
-def test_config(tmp_path, loaded_scenario_with_parsed_arguments, arguments):
- """Create the run configuration and check we've got the right values in there."""
- config = loaded_scenario_with_parsed_arguments.save_draws(return_config=True)
- assert config['scenario_seed'] == loaded_scenario_with_parsed_arguments.seed
- assert config['arguments'] == arguments
- assert len(config['draws']) == loaded_scenario_with_parsed_arguments.number_of_draws
-
-
-def test_runner(tmp_path, loaded_scenario_with_parsed_arguments, pop_size):
- """Check we can load the scenario from a configuration file."""
- config = loaded_scenario_with_parsed_arguments.save_draws(return_config=True)
- config_path = tmp_path / 'scenario.json'
- with open(config_path, 'w') as f:
- f.write(json.dumps(config, indent=2))
- runner = SampleRunner(config_path)
- scenario = runner.scenario
- assert isinstance(scenario, BaseScenario)
- assert scenario.__class__.__name__ == 'TestScenario'
- assert scenario.pop_size == pop_size
- assert runner.number_of_draws == loaded_scenario_with_parsed_arguments.number_of_draws
diff --git a/tests/test_simulation.py b/tests/test_simulation.py
deleted file mode 100644
index c26b501c47..0000000000
--- a/tests/test_simulation.py
+++ /dev/null
@@ -1,323 +0,0 @@
-from pathlib import Path
-from typing import Dict, List
-
-import numpy as np
-import pytest
-
-from tlo import Date, DateOffset, Module, Population, Simulation, logging
-from tlo.analysis.utils import merge_log_files, parse_log_file
-from tlo.methods.fullmodel import fullmodel
-from tlo.methods.healthsystem import HSI_Event, HSIEventQueueItem
-from tlo.simulation import (
- EventQueue,
- SimulationNotInitialisedError,
- SimulationPreviouslyInitialisedError,
-)
-
-
-def _check_basic_simulation_attributes_equal(
- simulation_1: Simulation, simulation_2: Simulation
-) -> None:
- for attribute in [
- "start_date",
- "end_date",
- "date",
- "show_progress_bar",
- "_custom_log_levels",
- "_seed",
- "_initialised",
- ]:
- assert getattr(simulation_1, attribute) == getattr(simulation_2, attribute)
-
-
-def _nested_dict_are_equal(nested_dict_1: dict, nested_dict_2: dict) -> bool:
- for key, value in nested_dict_1.items():
- if key not in nested_dict_2:
- return False
- if isinstance(value, np.ndarray):
- if not np.all(value == nested_dict_2[key]):
- return False
- elif isinstance(value, dict):
- if not _nested_dict_are_equal(value, nested_dict_2[key]):
- return False
- elif value != nested_dict_2[key]:
- return False
- return True
-
-
-def _check_random_state_equal(
- rng_1: np.random.RandomState, rng_2: np.random.RandomState
-) -> None:
- rng_state_1 = rng_1.get_state(legacy=False)
- rng_state_2 = rng_2.get_state(legacy=False)
- assert _nested_dict_are_equal(rng_state_1, rng_state_2)
-
-
-def _check_population_equal(population_1: Population, population_2: Population) -> None:
- assert population_1.initial_size == population_2.initial_size
- assert population_1.new_row.equals(population_2.new_row)
- assert population_1.new_rows.equals(population_2.new_rows)
- assert population_1.next_person_id == population_2.next_person_id
- assert population_1.props.equals(population_2.props)
-
-
-def _check_modules_are_equal(
- modules_dict_1: Dict[str, Module], modules_dict_2: Dict[str, Module]
-) -> None:
- for module_name, module_1 in modules_dict_1.items():
- assert module_name in modules_dict_2
- module_2 = modules_dict_2[module_name]
- assert module_2.PARAMETERS == module_1.PARAMETERS
- assert module_2.PROPERTIES == module_1.PROPERTIES
- _check_random_state_equal(module_1.rng, module_2.rng)
-
-
-def _check_event_queues_are_equal(
- event_queue_1: EventQueue, event_queue_2: EventQueue
-) -> None:
- assert len(event_queue_1) == len(event_queue_2)
- for (*date_priority_count_1, event_1), (*date_priority_count_2, event_2) in zip(
- event_queue_1.queue, event_queue_2.queue
- ):
- assert date_priority_count_1 == date_priority_count_2
- if isinstance(event_1.target, Population):
- # We don't check for equality of populations here as we do separately and
- # it would create a lot of redundancy to check for every event
- assert isinstance(event_2.target, Population)
- else:
- assert event_1.target == event_2.target
- assert event_1.priority == event_1.priority
- assert type(event_1.module) is type(event_2.module) # noqa: E721
-
-
-def _check_hsi_events_are_equal(hsi_event_1: HSI_Event, hsi_event_2: HSI_Event) -> None:
- if isinstance(hsi_event_1.target, Population):
- # We don't check for equality of populations here as we do separately and
- # it would create a lot of redundancy to check for every HSI event
- assert isinstance(hsi_event_2.target, Population)
- else:
- assert hsi_event_1.target == hsi_event_2.target
- assert hsi_event_1.module.name == hsi_event_2.module.name
- assert hsi_event_1.TREATMENT_ID == hsi_event_2.TREATMENT_ID
- assert hsi_event_1.ACCEPTED_FACILITY_LEVEL == hsi_event_2.ACCEPTED_FACILITY_LEVEL
- assert hsi_event_1.BEDDAYS_FOOTPRINT == hsi_event_2.BEDDAYS_FOOTPRINT
- assert (
- hsi_event_1._received_info_about_bed_days
- == hsi_event_2._received_info_about_bed_days
- )
- assert hsi_event_1.expected_time_requests == hsi_event_2.expected_time_requests
- assert hsi_event_1.facility_info == hsi_event_2.facility_info
-
-
-def _check_hsi_event_queues_are_equal(
- hsi_event_queue_1: List[HSIEventQueueItem],
- hsi_event_queue_2: List[HSIEventQueueItem],
-) -> None:
- assert len(hsi_event_queue_1) == len(hsi_event_queue_2)
- for hsi_event_queue_item_1, hsi_event_queue_item_2 in zip(
- hsi_event_queue_1, hsi_event_queue_2
- ):
- assert hsi_event_queue_item_1.priority == hsi_event_queue_item_2.priority
- assert hsi_event_queue_item_1.topen == hsi_event_queue_item_2.topen
- assert (
- hsi_event_queue_item_1.rand_queue_counter
- == hsi_event_queue_item_2.rand_queue_counter
- )
- assert hsi_event_queue_item_1.tclose == hsi_event_queue_item_2.tclose
- _check_hsi_events_are_equal(
- hsi_event_queue_item_1.hsi_event, hsi_event_queue_item_2.hsi_event
- )
-
-
-def _check_simulations_are_equal(
- simulation_1: Simulation, simulation_2: Simulation
-) -> None:
- _check_basic_simulation_attributes_equal(simulation_1, simulation_2)
- _check_modules_are_equal(simulation_1.modules, simulation_2.modules)
- _check_random_state_equal(simulation_1.rng, simulation_2.rng)
- _check_event_queues_are_equal(simulation_1.event_queue, simulation_2.event_queue)
- _check_hsi_event_queues_are_equal(
- simulation_1.modules["HealthSystem"].HSI_EVENT_QUEUE,
- simulation_2.modules["HealthSystem"].HSI_EVENT_QUEUE,
- )
- _check_population_equal(simulation_1.population, simulation_2.population)
-
-
-@pytest.fixture(scope="module")
-def resource_file_path():
- return Path(__file__).parents[1] / "resources"
-
-
-@pytest.fixture(scope="module")
-def initial_population_size():
- return 5000
-
-
-@pytest.fixture(scope="module")
-def start_date():
- return Date(2010, 1, 1)
-
-
-@pytest.fixture(scope="module")
-def end_date(start_date):
- return start_date + DateOffset(days=180)
-
-
-@pytest.fixture(scope="module")
-def intermediate_date(start_date, end_date):
- return start_date + (end_date - start_date) / 2
-
-
-@pytest.fixture(scope="module")
-def logging_custom_levels():
- return {"*": logging.INFO}
-
-
-def _simulation_factory(
- output_directory, start_date, seed, resource_file_path, logging_custom_levels
-):
- log_config = {
- "filename": "test",
- "directory": output_directory,
- "custom_levels": logging_custom_levels,
- }
- simulation = Simulation(
- start_date=start_date,
- seed=seed,
- log_config=log_config,
- )
- simulation.register(
- *fullmodel(
- resourcefilepath=resource_file_path,
- )
- )
- return simulation
-
-
-@pytest.fixture
-def simulation(tmp_path, start_date, seed, resource_file_path, logging_custom_levels):
- return _simulation_factory(
- tmp_path, start_date, seed, resource_file_path, logging_custom_levels
- )
-
-
-@pytest.fixture(scope="module")
-def simulated_simulation(
- tmp_path_factory,
- start_date,
- end_date,
- seed,
- resource_file_path,
- initial_population_size,
- logging_custom_levels,
-):
- tmp_path = tmp_path_factory.mktemp("simulated_simulation")
- simulation = _simulation_factory(
- tmp_path, start_date, seed, resource_file_path, logging_custom_levels
- )
- simulation.make_initial_population(n=initial_population_size)
- simulation.simulate(end_date=end_date)
- return simulation
-
-
-def test_save_to_pickle_creates_file(tmp_path, simulation):
- pickle_path = tmp_path / "simulation.pkl"
- simulation.save_to_pickle(pickle_path=pickle_path)
- assert pickle_path.exists()
-
-
-def test_save_load_pickle_after_initialising(
- tmp_path, simulation, initial_population_size
-):
- simulation.make_initial_population(n=initial_population_size)
- simulation.initialise(end_date=simulation.start_date)
- pickle_path = tmp_path / "simulation.pkl"
- simulation.save_to_pickle(pickle_path=pickle_path)
- loaded_simulation = Simulation.load_from_pickle(pickle_path)
- _check_simulations_are_equal(simulation, loaded_simulation)
-
-
-def test_save_load_pickle_after_simulating(tmp_path, simulated_simulation):
- pickle_path = tmp_path / "simulation.pkl"
- simulated_simulation.save_to_pickle(pickle_path=pickle_path)
- loaded_simulation = Simulation.load_from_pickle(pickle_path)
- _check_simulations_are_equal(simulated_simulation, loaded_simulation)
-
-
-def _check_parsed_logs_are_equal(
- log_path_1: Path,
- log_path_2: Path,
- module_name_key_pairs_to_skip: set[tuple[str, str]],
-) -> None:
- logs_dict_1 = parse_log_file(log_path_1)
- logs_dict_2 = parse_log_file(log_path_2)
- assert logs_dict_1.keys() == logs_dict_2.keys()
- for module_name in logs_dict_1.keys():
- module_logs_1 = logs_dict_1[module_name]
- module_logs_2 = logs_dict_2[module_name]
- assert module_logs_1.keys() == module_logs_2.keys()
- for key in module_logs_1:
- if key == "_metadata":
- assert module_logs_1[key] == module_logs_2[key]
- elif (module_name, key) not in module_name_key_pairs_to_skip:
- assert module_logs_1[key].equals(module_logs_2[key])
-
-
-@pytest.mark.slow
-def test_continuous_and_interrupted_simulations_equal(
- tmp_path,
- simulation,
- simulated_simulation,
- initial_population_size,
- intermediate_date,
- end_date,
- logging_custom_levels,
-):
- simulation.make_initial_population(n=initial_population_size)
- simulation.initialise(end_date=end_date)
- simulation.run_simulation_to(to_date=intermediate_date)
- pickle_path = tmp_path / "simulation.pkl"
- simulation.save_to_pickle(pickle_path=pickle_path)
- simulation.close_output_file()
- log_config = {
- "filename": "test_continued",
- "directory": tmp_path,
- "custom_levels": logging_custom_levels,
- }
- interrupted_simulation = Simulation.load_from_pickle(pickle_path, log_config)
- interrupted_simulation.run_simulation_to(to_date=end_date)
- interrupted_simulation.finalise()
- _check_simulations_are_equal(simulated_simulation, interrupted_simulation)
- merged_log_path = tmp_path / "concatenated.log"
- merge_log_files(
- simulation.log_filepath, interrupted_simulation.log_filepath, merged_log_path
- )
- _check_parsed_logs_are_equal(
- simulated_simulation.log_filepath, merged_log_path, {("tlo.simulation", "info")}
- )
-
-
-def test_run_simulation_to_past_end_date_raises(
- simulation, initial_population_size, end_date
-):
- simulation.make_initial_population(n=initial_population_size)
- simulation.initialise(end_date=end_date)
- with pytest.raises(ValueError, match="after simulation end date"):
- simulation.run_simulation_to(to_date=end_date + DateOffset(days=1))
-
-
-def test_run_simulation_without_initialisation_raises(
- simulation, initial_population_size, end_date
-):
- simulation.make_initial_population(n=initial_population_size)
- with pytest.raises(SimulationNotInitialisedError):
- simulation.run_simulation_to(to_date=end_date)
-
-
-def test_initialise_simulation_twice_raises(
- simulation, initial_population_size, end_date
-):
- simulation.make_initial_population(n=initial_population_size)
- simulation.initialise(end_date=end_date)
- with pytest.raises(SimulationPreviouslyInitialisedError):
- simulation.initialise(end_date=end_date)
diff --git a/tests/test_stunting.py b/tests/test_stunting.py
index f41bab7c78..4b11db1512 100644
--- a/tests/test_stunting.py
+++ b/tests/test_stunting.py
@@ -235,17 +235,15 @@ def test_routine_assessment_for_chronic_undernutrition_if_stunted_and_correctly_
person_id = 0
df.loc[person_id, 'age_years'] = 2
df.loc[person_id, "un_HAZ_category"] = "-3<=HAZ<-2"
+ patient_details = sim.population.row_in_readonly_form(person_id)
# Make the probability of stunting checking/diagnosis as 1.0
sim.modules['Stunting'].parameters['prob_stunting_diagnosed_at_generic_appt'] = 1.0
- with sim.population.individual_properties(person_id) as individual_properties:
- # Subject the person to `do_at_generic_first_appt`
- sim.modules["Stunting"].do_at_generic_first_appt(
- person_id=person_id,
- individual_properties=individual_properties,
- schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event,
- )
+ # Subject the person to `do_at_generic_first_appt`
+ sim.modules["Stunting"].do_at_generic_first_appt(
+ patient_id=person_id, patient_details=patient_details
+ )
# Check that there is an HSI scheduled for this person
hsi_event_scheduled = [
@@ -304,17 +302,13 @@ def test_routine_assessment_for_chronic_undernutrition_if_stunted_but_no_checkin
person_id = 0
df.loc[person_id, 'age_years'] = 2
df.loc[person_id, "un_HAZ_category"] = "HAZ<-3"
+ patient_details = sim.population.row_in_readonly_form(person_id)
# Make the probability of stunting checking/diagnosis as 0.0
sim.modules['Stunting'].parameters['prob_stunting_diagnosed_at_generic_appt'] = 0.0
- with sim.population.individual_properties(person_id) as individual_properties:
- # Subject the person to `do_at_generic_first_appt`
- sim.modules['Stunting'].do_at_generic_first_appt(
- person_id=person_id,
- individual_properties=individual_properties,
- schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event,
- )
+ # Subject the person to `do_at_generic_first_appt`
+ sim.modules["Stunting"].do_at_generic_first_appt(patient_id=person_id, patient_details=patient_details)
# Check that there is no HSI scheduled for this person
hsi_event_scheduled = [
@@ -326,13 +320,10 @@ def test_routine_assessment_for_chronic_undernutrition_if_stunted_but_no_checkin
# Then make the probability of stunting checking/diagnosis 1.0
# and check the HSI is scheduled for this person
- sim.modules['Stunting'].parameters["prob_stunting_diagnosed_at_generic_appt"] = 1.0
- with sim.population.individual_properties(person_id) as individual_properties:
- sim.modules['Stunting'].do_at_generic_first_appt(
- person_id=person_id,
- individual_properties=individual_properties,
- schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event,
- )
+ sim.modules["Stunting"].parameters["prob_stunting_diagnosed_at_generic_appt"] = 1.0
+ sim.modules["Stunting"].do_at_generic_first_appt(
+ patient_id=person_id, patient_details=patient_details
+ )
hsi_event_scheduled = [
ev[1]
for ev in sim.modules["HealthSystem"].find_events_for_person(person_id)
@@ -355,13 +346,10 @@ def test_routine_assessment_for_chronic_undernutrition_if_not_stunted(seed):
person_id = 0
df.loc[person_id, 'age_years'] = 2
df.loc[person_id, 'un_HAZ_category'] = 'HAZ>=-2'
- with sim.population.individual_properties(person_id) as individual_properties:
- # Subject the person to `do_at_generic_first_appt`
- sim.modules["Stunting"].do_at_generic_first_appt(
- person_id=person_id,
- individual_properties=individual_properties,
- schedule_hsi_event=sim.modules["HealthSystem"].schedule_hsi_event,
- )
+ patient_details = sim.population.row_in_readonly_form(person_id)
+
+ # Subject the person to `do_at_generic_first_appt`
+ sim.modules["Stunting"].do_at_generic_first_appt(patient_id=person_id, patient_details=patient_details)
# Check that there is no HSI scheduled for this person
hsi_event_scheduled = [
diff --git a/tests/test_symptommanager.py b/tests/test_symptommanager.py
index 73ea7619d0..85c7156902 100644
--- a/tests/test_symptommanager.py
+++ b/tests/test_symptommanager.py
@@ -1,8 +1,5 @@
-from __future__ import annotations
-
import os
from pathlib import Path
-from typing import TYPE_CHECKING, List
import pytest
from pandas import DateOffset
@@ -27,9 +24,6 @@
SymptomManager_SpuriousSymptomOnset,
)
-if TYPE_CHECKING:
- from tlo.methods.symptommanager import SymptomManager
-
try:
resourcefilepath = Path(os.path.dirname(__file__)) / '../resources'
except NameError:
@@ -193,9 +187,8 @@ def test_adding_quering_and_removing_symptoms(seed):
assert set(has_symp) == set(ids)
for person_id in ids:
- assert symp in sim.modules["SymptomManager"].has_what(
- person_id=person_id, disease_module=sim.modules["Mockitis"]
- )
+ assert symp in sim.modules['SymptomManager'].has_what(person_id=person_id,
+ disease_module=sim.modules['Mockitis'])
# Check cause of the symptom:
for person in ids:
@@ -210,103 +203,6 @@ def test_adding_quering_and_removing_symptoms(seed):
assert list() == sim.modules['SymptomManager'].who_has(symp)
-@pytest.mark.parametrize(
- "supply_disease_module",
- [
- pytest.param(False, id="disease_module kwarg NOT supplied"),
- pytest.param(True, id="disease_module kwarg supplied"),
- ],
-)
-def test_has_what_via_individual_properties(seed, supply_disease_module: bool):
- """
- Test that the has_what method returns the same symptoms for an individual
- when supplied a person_id and the individual_properties context for that
- same person.
-
- Test the case when the optional disease_module kwarg is supplied as well.
-
- We will create 3 'dummy' symptoms and select 8 individuals in the
- population to infect with these symptoms; in the following combinations:
-
- id has_symp1 has_symp2 has_symp3
- 0 1 1 1
- 1 1 1 0
- 2 1 0 1
- 3 1 0 0
- 4 0 1 1
- 5 0 1 0
- 6 0 0 1
- 7 0 0 0
-
- We will then assert that has_what returns the expected symptoms for the
- individuals, and that supplying either the person_id keyword or the
- individual_properties keyword gives the same answer.
- """
- sim = Simulation(start_date=start_date, seed=seed)
- sim.register(
- demography.Demography(resourcefilepath=resourcefilepath),
- enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath),
- healthsystem.HealthSystem(resourcefilepath=resourcefilepath, disable=True),
- symptommanager.SymptomManager(resourcefilepath=resourcefilepath),
- healthseekingbehaviour.HealthSeekingBehaviour(
- resourcefilepath=resourcefilepath
- ),
- simplified_births.SimplifiedBirths(resourcefilepath=resourcefilepath),
- mockitis.Mockitis(),
- chronicsyndrome.ChronicSyndrome(),
- )
- disease_module: mockitis.Mockitis = sim.modules["Mockitis"]
- symptom_manager: SymptomManager = sim.modules["SymptomManager"]
-
- # Generate the symptoms and select the people to infect
- n_symptoms = 3
- n_patients = 2 ** n_symptoms
- symptoms = [f"test_symptom{i}" for i in range(n_symptoms)]
- symptom_manager.register_symptom(*[Symptom(name=symptom) for symptom in symptoms])
-
- # Create the initial population after generating extra symptoms, so that they are registered
- sim.make_initial_population(n=popsize)
- df = sim.population.props
-
- # Infect the people with the corresponding symptoms
- persons_infected_with: List[int] = [
- id for id in sim.rng.choice(list(df.index[df.is_alive]), n_patients)
- ]
- for i, id in enumerate(persons_infected_with):
- bin_rep = format(i, f"0{n_symptoms}b")
- for symptom_number, digit in enumerate(bin_rep):
- if digit == "1":
- symptom_manager.change_symptom(
- symptom_string=symptoms[symptom_number],
- person_id=[id],
- add_or_remove="+",
- disease_module=disease_module,
- )
-
- # Now check that has_what returns the same (correct!) arguments when supplied with
- # individual_properties and person_id.
- for person_id in persons_infected_with:
- symptoms_via_pid = symptom_manager.has_what(
- person_id=person_id,
- disease_module=disease_module if supply_disease_module else None,
- )
- with sim.population.individual_properties(
- person_id, read_only=True
- ) as individual_properties:
- symptoms_via_iprops = symptom_manager.has_what(
- individual_details=individual_properties,
- disease_module=disease_module if supply_disease_module else None,
- )
-
- # Assert all returned symptoms are in agreement
- assert len(symptoms_via_pid) == len(
- symptoms_via_iprops
- ), "Method does not return same number of symptoms."
- assert set(symptoms_via_pid) == set(
- symptoms_via_iprops
- ), "Method does not return the same symptoms"
-
-
def test_baby_born_has_no_symptoms(seed):
sim = Simulation(start_date=start_date, seed=seed)
@@ -331,7 +227,7 @@ def test_baby_born_has_no_symptoms(seed):
person_id = sim.do_birth(mother_id)
# check that the new person does not have symptoms:
- assert [] == sim.modules['SymptomManager'].has_what(person_id=person_id)
+ assert [] == sim.modules['SymptomManager'].has_what(person_id)
def test_auto_onset_symptom(seed):
@@ -354,7 +250,7 @@ def test_auto_onset_symptom(seed):
sim.population.props.loc[person_id, 'is_alive'] = True
for symptom in sm.symptom_names:
sim.population.props.loc[person_id, sm.get_column_name_for_symptom(symptom)] = 0
- assert 0 == len(sm.has_what(person_id=person_id))
+ assert 0 == len(sm.has_what(person_id))
def get_events_in_sim():
return [ev for ev in sim.event_queue.queue if (person_id in ev[3].person_id)]
@@ -377,7 +273,7 @@ def get_events_in_sim():
)
# check that the symptom is not imposed
- assert 0 == len(sm.has_what(person_id=person_id))
+ assert 0 == len(sm.has_what(person_id))
# get the future events for this person (should be just the auto-onset event)
assert 1 == len(get_events_in_sim())
@@ -389,7 +285,7 @@ def get_events_in_sim():
# run the events and check for the changing of symptoms
sim.date = date_of_onset
onset[3].apply(sim.population)
- assert symptom_string in sm.has_what(person_id=person_id)
+ assert symptom_string in sm.has_what(person_id)
# get the future events for this person (should now include the auto-resolve event)
assert 2 == len(get_events_in_sim())
@@ -399,7 +295,7 @@ def get_events_in_sim():
assert isinstance(resolve[3], SymptomManager_AutoResolveEvent)
resolve[3].apply(sim.population)
- assert 0 == len(sm.has_what(person_id=person_id))
+ assert 0 == len(sm.has_what(person_id))
def test_nonemergency_spurious_symptoms_during_simulation(seed):
@@ -608,26 +504,13 @@ def test_has_what(
df.is_alive
& (df[symptom_manager.get_column_name_for_symptom(symptom)] > 0)
][0]
- assert symptom in symptom_manager.has_what(person_id=person_with_symptom)
+ assert symptom in symptom_manager.has_what(person_with_symptom)
person_without_symptom = df.index[
df.is_alive
& (df[symptom_manager.get_column_name_for_symptom(symptom)] == 0)
][0]
- assert symptom not in symptom_manager.has_what(person_id=person_without_symptom)
-
- # Do the same checks but using an IndividualDetails context
- with simulation.population.individual_properties(
- person_with_symptom, read_only=True
- ) as with_symptom_properties:
- assert symptom in symptom_manager.has_what(
- individual_details=with_symptom_properties
- )
- with simulation.population.individual_properties(
- person_without_symptom, read_only=True
- ) as without_symptom_properties:
- assert symptom not in symptom_manager.has_what(
- individual_details=without_symptom_properties
- )
+ assert symptom not in symptom_manager.has_what(person_without_symptom)
+
def test_has_what_disease_module(
symptom_manager, disease_module, disease_module_symptoms, simulation
@@ -639,16 +522,12 @@ def test_has_what_disease_module(
df.is_alive
& (df[symptom_manager.get_column_name_for_symptom(symptom)] > 0)
][0]
- assert symptom in symptom_manager.has_what(
- person_id=person_with_symptom, disease_module=disease_module
- )
+ assert symptom in symptom_manager.has_what(person_with_symptom, disease_module)
person_without_symptom = df.index[
df.is_alive
& (df[symptom_manager.get_column_name_for_symptom(symptom)] == 0)
][0]
- assert symptom not in symptom_manager.has_what(
- person_id=person_without_symptom, disease_module=disease_module
- )
+ assert symptom not in symptom_manager.has_what(person_without_symptom, disease_module)
def test_have_what(
diff --git a/tests/test_tb.py b/tests/test_tb.py
index 66d5abd60e..0434c70069 100644
--- a/tests/test_tb.py
+++ b/tests/test_tb.py
@@ -576,7 +576,7 @@ def test_children_referrals(seed):
duration_in_days=None,
)
- assert set(sim.modules['SymptomManager'].has_what(person_id=person_id)) == symptom_list
+ assert set(sim.modules['SymptomManager'].has_what(person_id)) == symptom_list
# run HSI_Tb_ScreeningAndRefer and check outcomes
sim.modules['HealthSystem'].schedule_hsi_event(
@@ -1036,7 +1036,7 @@ def test_hsi_scheduling(seed):
duration_in_days=None,
)
- assert set(sim.modules['SymptomManager'].has_what(person_id=person_id)) == symptom_list
+ assert set(sim.modules['SymptomManager'].has_what(person_id)) == symptom_list
hsi_event = tb.HSI_Tb_ScreeningAndRefer(person_id=person_id, module=sim.modules['Tb'])
hsi_event.run(squeeze_factor=0)
@@ -1080,7 +1080,7 @@ def test_hsi_scheduling(seed):
duration_in_days=None,
)
- assert set(sim.modules['SymptomManager'].has_what(person_id=person_id)) == symptom_list
+ assert set(sim.modules['SymptomManager'].has_what(person_id)) == symptom_list
hsi_event = tb.HSI_Tb_ScreeningAndRefer(person_id=person_id, module=sim.modules['Tb'])
hsi_event.run(squeeze_factor=0)
@@ -1125,7 +1125,7 @@ def test_hsi_scheduling(seed):
duration_in_days=None,
)
- assert set(sim.modules['SymptomManager'].has_what(person_id=person_id)) == symptom_list
+ assert set(sim.modules['SymptomManager'].has_what(person_id)) == symptom_list
hsi_event = tb.HSI_Tb_ScreeningAndRefer(person_id=person_id, module=sim.modules['Tb'])
hsi_event.run(squeeze_factor=0)
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 1022c95010..02ae63b7ba 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,7 +1,6 @@
"""Unit tests for utility functions."""
import os
import pickle
-import shutil
import string
import types
from pathlib import Path
@@ -15,7 +14,7 @@
from tlo import Date, Simulation
from tlo.analysis.utils import parse_log_file
from tlo.methods import demography
-from tlo.util import DEFAULT_MOTHER_ID, convert_excel_files_to_csv, read_csv_files
+from tlo.util import DEFAULT_MOTHER_ID
path_to_files = Path(os.path.dirname(__file__))
@@ -318,148 +317,3 @@ def check_hash_is_valid(dfh):
# check hash differs for different dataframes
if not dataframes[i].equals(dataframes[j]):
assert df_hash != tlo.util.hash_dataframe(dataframes[j])
-
-
-def copy_files_to_temporal_directory_and_return_path(tmpdir):
- """ copy resource files in tests/resources to a temporal directory and return its path
-
- :param tmpdir: path to a temporal directory
-
- """
- resource_filepath = path_to_files / 'resources'
- tmpdir_resource_filepath = Path(tmpdir / 'resources')
- shutil.copytree(resource_filepath, tmpdir_resource_filepath)
- return tmpdir_resource_filepath
-
-
-def test_read_csv_method_with_no_file(tmpdir):
- """ read csv method when no file name is supplied
- i) should return dictionary.
- ii) dictionary keys should match csv file names in resource folder
- iii) all dictionary values should be dataframes
-
- :param tmpdir: path to a temporal directory
-
- """
- tmpdir_resource_filepath = copy_files_to_temporal_directory_and_return_path(tmpdir)
- file_names = [csv_file_path.stem for csv_file_path in tmpdir_resource_filepath.rglob("*.csv")]
- df_no_files = read_csv_files(tmpdir_resource_filepath)
- assert isinstance(df_no_files, dict)
- assert set(df_no_files.keys()) == set(file_names)
- assert all(isinstance(value, pd.DataFrame) for value in df_no_files.values())
-
-
-def test_read_csv_method_with_one_file(tmpdir):
- """ test read csv method when one file name is supplied. should return a dataframe
- :param tmpdir: path to a temporal directory
-
- """
- tmpdir_resource_filepath = copy_files_to_temporal_directory_and_return_path(tmpdir)
- df = read_csv_files(tmpdir_resource_filepath, files=['df_at_healthcareseeking'])
- assert isinstance(df, pd.DataFrame)
-
-
-def test_read_csv_method_with_multiple_files(tmpdir):
- """ read csv method when multiple file names are supplied.
- i) should return dictionary.
- ii) dictionary keys should match supplied file names
- iii) all dictionary values should be dataframes
-
- :param tmpdir: path to a temporal directory
-
- """
- tmpdir_resource_filepath = copy_files_to_temporal_directory_and_return_path(tmpdir)
- file_names = ['df_at_healthcareseeking', 'df_at_init_of_lifestyle']
- df_dict = read_csv_files(tmpdir_resource_filepath, files=file_names)
- assert isinstance(df_dict, dict)
- assert set(df_dict.keys()) == set(file_names)
- for _key, dataframe in df_dict.items():
- assert isinstance(dataframe, pd.DataFrame)
-
-
-def test_read_csv_method_output_matches_previously_used_read_excel(tmpdir):
- """ check read from csv method produces same output as the read Excel file
- :param tmpdir: path to a temporal directory
-
- """
- tmpdir_resource_filepath = copy_files_to_temporal_directory_and_return_path(tmpdir)
- excel_file_path = Path(tmpdir_resource_filepath
- / 'ResourceFile_test_convert_to_csv/ResourceFile_test_convert_to_csv.xlsx')
- xls = pd.ExcelFile(excel_file_path)
- sheet_names = xls.sheet_names
- # convert the above Excel file into csv equivalent. we will use the newly converted files to determine if
- # loading parameters from Excel file will be equal to loading parameters from the converted csv files
- convert_excel_files_to_csv(folder=Path(tmpdir_resource_filepath / 'ResourceFile_test_convert_to_csv'),
- files=[excel_file_path.name])
-
- # get excel sheet names
- df_excel = pd.read_excel(xls, sheet_name=sheet_names)
-
- # read newly converted csv files using read_csv_files method
- df_csv = read_csv_files(Path(str(excel_file_path).split('.')[0]),
- files=sheet_names)
-
- # dictionary keys from both dataframe dictionaries should match
- assert isinstance(df_excel, dict) and isinstance(df_csv, dict)
- assert df_excel.keys() == df_csv.keys()
- for key in df_excel:
- assert df_excel[key].astype(str).equals(df_csv[key].astype(str))
-
-
-def test_convert_excel_files_method(tmpdir):
- """ Test converting Excel files to csv equivalent is done as expected
-
- 1) Excel file name should become the name of the folder containing the newly converted csv files
- 2) Excel file sheet names should become csv file names
- 3) if files are given, the function should only convert to excel only those given files in a folder
- 4) if no files are given, all Excel files in the parent folder and subsequent folders within the parent folder
- should get converted to csv files
-
- """
-
- def check_logic_of_converting_excel_files_to_csv_files(folder: Path, files: list) -> None:
- """ check converting Excel files to csv files is done as expected
- 1) check that a new directory to hold the newly created csv files has been created
- 2) check that this new directory name matches the Excel file name it has been created from
- 3) check csv files are created and that the csv names should match sheet names of an Excel file they
- have been created from
- """
- # check that the above function has created a folder named `ResourceFile_load-parameters`(name of the Excel
- # file) and a csv file named `parameter_values` (Excel file sheet name).
- excel_file_paths = [folder / file for file in files]
-
- for excel_file_path in excel_file_paths:
- xl = pd.ExcelFile(excel_file_path)
- path_to_new_directory = excel_file_path.with_suffix("")
- # new folder should be created
- assert path_to_new_directory.exists() and path_to_new_directory.is_dir()
- # the new folder name should be the same as the Excel file name
- assert excel_file_path.stem == path_to_new_directory.name
- for sheet_name in xl.sheet_names:
- path_to_new_file = Path(path_to_new_directory / f'{sheet_name}.csv')
- # new csv file(s) should be created with name(s) resembling sheet name(s) in excel file
- assert path_to_new_file.exists() and path_to_new_file.is_file()
- assert sheet_name == path_to_new_file.name.split('.')[0]
-
-
- # get resource file path
- resourcefilepath = path_to_files / 'resources'
- tmpdir_resourcefilepath = Path(tmpdir/'resources')
- shutil.copytree(resourcefilepath, tmpdir_resourcefilepath)
-
- # check convert to csv logic when a list of file name(s) is given
- excel_file = ['ResourceFile_load-parameters.xlsx']
- convert_excel_files_to_csv(tmpdir_resourcefilepath, files=excel_file)
- # check new folder containing csv file is created. The folder name and csv file name should resemble the supplied
- # Excel file name and sheet name respectively
- check_logic_of_converting_excel_files_to_csv_files(tmpdir_resourcefilepath, files=excel_file)
-
- # check convert to csv logic when no list of file name(s) is given
- excel_files = [file for file in tmpdir_resourcefilepath.rglob("*.xlsx")]
- if excel_files is None:
- excel_files = excel_file
-
- convert_excel_files_to_csv(tmpdir_resourcefilepath)
- # check behaviours are as expected. New folders containing csv files should be created with names resembling the
- # Excel file they were created from
- check_logic_of_converting_excel_files_to_csv_files(tmpdir_resourcefilepath, excel_files)
diff --git a/tox.ini b/tox.ini
index 4c63f7ea0f..94949bd6d8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -41,13 +41,6 @@ deps =
pytest
pytest-cov
-[testenv:py3-latest]
-deps =
- pytest
- pytest-xdist
-commands =
- {posargs:pytest -n auto -vv tests}
-
[testenv:spell]
setenv =
SPELLCHECK=1
@@ -73,15 +66,11 @@ commands =
python docs/tlo_data_sources.py
; Generate contributors page
python docs/tlo_contributors.py
- ; Generate publications page
- python docs/tlo_publications.py
; Generate resources files page
python docs/tlo_resources.py
; Generate HSI events listing
python src/tlo/analysis/hsi_events.py --output-file docs/_hsi_events.rst --output-format rst-list
python src/tlo/analysis/hsi_events.py --output-file docs/hsi_events.csv --output-format csv
- ; Generate parameters listing
- python docs/tlo_parameters.py {toxinidir}{/}resources {toxinidir}{/}docs{/}parameters.rst
sphinx-build {posargs:-E} -b html docs dist/docs
-sphinx-build -b linkcheck docs dist/docs
@@ -104,7 +93,7 @@ commands =
twine check dist/*.tar.gz dist/*.whl
; ignore that _version.py file generated by setuptools_scm is not tracked by VCS
check-manifest --ignore **/_version.py {toxinidir}
- ruff check src tests
+ ruff src tests
isort --check-only --diff src tests
pylint src tests
python {toxinidir}/src/scripts/automation/update_citation.py --check
@@ -141,26 +130,9 @@ commands = python {toxinidir}/src/scripts/automation/update_citation.py
skip_install = true
deps = pyyaml
-[testenv:update-publications]
-commands = python {toxinidir}/docs/tlo_publications.py --update-from-zotero
-skip_install = true
-deps =
- pybtex
- requests
-
[testenv:requirements]
commands =
pip-compile --output-file {toxinidir}/requirements/base.txt
pip-compile --extra dev --output-file {toxinidir}/requirements/dev.txt
skip_install = true
deps = pip-tools
-
-[testenv:markslow]
-deps =
- -r{toxinidir}/requirements/base.txt
- pytest
- pytest-json-report
- redbaron
-commands =
- pytest tests --json-report --json-report-file {toxinidir}/test-report.json --json-report-omit collectors log traceback streams warnings
- python {toxinidir}/src/scripts/automation/mark_slow_tests.py --json-test-report-path test-report.json