Convert excel files to csv files epi_hiv_malaria_measles_schisto_tb (#…

…1484) * converting Excel files to csv files * use read csv files method * test_hiv, use read csv files method * remove unused import(import os) * added a method to parse output * updated file selection * use parse csv data types function in util
UCL · Dec 17, 2024 · 2e6cb9c · 2e6cb9c
1 parent d60c3b6
commit 2e6cb9c
Show file tree

Hide file tree

Showing 83 changed files with 248 additions and 60 deletions.
diff --git a/resources/ResourceFile_EPI_WHO_estimates.xlsx b/resources/ResourceFile_EPI_WHO_estimates.xlsx
diff --git a/resources/ResourceFile_EPI_WHO_estimates/WHO_estimates.csv b/resources/ResourceFile_EPI_WHO_estimates/WHO_estimates.csv
diff --git a/resources/ResourceFile_EPI_WHO_estimates/parameters.csv b/resources/ResourceFile_EPI_WHO_estimates/parameters.csv
diff --git a/resources/ResourceFile_EPI_WHO_estimates/vaccine_schedule.csv b/resources/ResourceFile_EPI_WHO_estimates/vaccine_schedule.csv
diff --git a/resources/ResourceFile_HIV.xlsx b/resources/ResourceFile_HIV.xlsx
diff --git a/resources/ResourceFile_HIV/DHS_prevalence.csv b/resources/ResourceFile_HIV/DHS_prevalence.csv
diff --git a/resources/ResourceFile_HIV/LHC_samples.csv b/resources/ResourceFile_HIV/LHC_samples.csv
diff --git a/resources/ResourceFile_HIV/MPHIA_incidence2020.csv b/resources/ResourceFile_HIV/MPHIA_incidence2020.csv
diff --git a/resources/ResourceFile_HIV/MPHIA_prevalence_art2020.csv b/resources/ResourceFile_HIV/MPHIA_prevalence_art2020.csv
diff --git a/resources/ResourceFile_HIV/MoH_CPT_IPT2020.csv b/resources/ResourceFile_HIV/MoH_CPT_IPT2020.csv
diff --git a/resources/ResourceFile_HIV/MoH_number_art.csv b/resources/ResourceFile_HIV/MoH_number_art.csv
diff --git a/resources/ResourceFile_HIV/MoH_numbers_tests.csv b/resources/ResourceFile_HIV/MoH_numbers_tests.csv
diff --git a/resources/ResourceFile_HIV/art_coverage.csv b/resources/ResourceFile_HIV/art_coverage.csv
diff --git a/resources/ResourceFile_HIV/calibration_from_aids_info.csv b/resources/ResourceFile_HIV/calibration_from_aids_info.csv
diff --git a/resources/ResourceFile_HIV/children0_14_prev_AIDSinfo.csv b/resources/ResourceFile_HIV/children0_14_prev_AIDSinfo.csv
diff --git a/resources/ResourceFile_HIV/hiv_prevalence.csv b/resources/ResourceFile_HIV/hiv_prevalence.csv
diff --git a/resources/ResourceFile_HIV/parameters.csv b/resources/ResourceFile_HIV/parameters.csv
diff --git a/resources/ResourceFile_HIV/scaleup_parameters.csv b/resources/ResourceFile_HIV/scaleup_parameters.csv
diff --git a/resources/ResourceFile_HIV/spectrum_treatment_cascade.csv b/resources/ResourceFile_HIV/spectrum_treatment_cascade.csv
diff --git a/resources/ResourceFile_HIV/time_since_infection_at_baselin.csv b/resources/ResourceFile_HIV/time_since_infection_at_baselin.csv
diff --git a/resources/ResourceFile_HIV/unaids_infections_art2021.csv b/resources/ResourceFile_HIV/unaids_infections_art2021.csv
diff --git a/resources/ResourceFile_HIV/unaids_mortality_dalys2021.csv b/resources/ResourceFile_HIV/unaids_mortality_dalys2021.csv
diff --git a/resources/ResourceFile_HIV/unaids_pmtct2021.csv b/resources/ResourceFile_HIV/unaids_pmtct2021.csv
diff --git a/resources/ResourceFile_HIV/unaids_program_perf.csv b/resources/ResourceFile_HIV/unaids_program_perf.csv
diff --git a/resources/ResourceFile_Measles.xlsx b/resources/ResourceFile_Measles.xlsx
diff --git a/resources/ResourceFile_Measles/beta.csv b/resources/ResourceFile_Measles/beta.csv
diff --git a/resources/ResourceFile_Measles/cfr.csv b/resources/ResourceFile_Measles/cfr.csv
diff --git a/resources/ResourceFile_Measles/parameters.csv b/resources/ResourceFile_Measles/parameters.csv
diff --git a/resources/ResourceFile_Measles/symptoms.csv b/resources/ResourceFile_Measles/symptoms.csv
diff --git a/resources/ResourceFile_Schisto.xlsx b/resources/ResourceFile_Schisto.xlsx
diff --git a/resources/ResourceFile_Schisto/DALYs.csv b/resources/ResourceFile_Schisto/DALYs.csv
diff --git a/resources/ResourceFile_Schisto/District_Params_haematobium.csv b/resources/ResourceFile_Schisto/District_Params_haematobium.csv
diff --git a/resources/ResourceFile_Schisto/District_Params_mansoni.csv b/resources/ResourceFile_Schisto/District_Params_mansoni.csv
diff --git a/resources/ResourceFile_Schisto/MDA_historical_Coverage.csv b/resources/ResourceFile_Schisto/MDA_historical_Coverage.csv
diff --git a/resources/ResourceFile_Schisto/MDA_prognosed_Coverage.csv b/resources/ResourceFile_Schisto/MDA_prognosed_Coverage.csv
diff --git a/resources/ResourceFile_Schisto/Parameters.csv b/resources/ResourceFile_Schisto/Parameters.csv
diff --git a/resources/ResourceFile_Schisto/Symptoms.csv b/resources/ResourceFile_Schisto/Symptoms.csv
diff --git a/resources/ResourceFile_TB.xlsx b/resources/ResourceFile_TB.xlsx
diff --git a/resources/ResourceFile_TB/IPTdistricts.csv b/resources/ResourceFile_TB/IPTdistricts.csv
diff --git a/resources/ResourceFile_TB/NTP2019.csv b/resources/ResourceFile_TB/NTP2019.csv
diff --git a/resources/ResourceFile_TB/WHO_activeTB2023.csv b/resources/ResourceFile_TB/WHO_activeTB2023.csv
diff --git a/resources/ResourceFile_TB/WHO_latentTB2017.csv b/resources/ResourceFile_TB/WHO_latentTB2017.csv
diff --git a/resources/ResourceFile_TB/WHO_mdrTB2017.csv b/resources/ResourceFile_TB/WHO_mdrTB2017.csv
diff --git a/resources/ResourceFile_TB/WHO_tx_success_rates2021.csv b/resources/ResourceFile_TB/WHO_tx_success_rates2021.csv
diff --git a/resources/ResourceFile_TB/additional_params.csv b/resources/ResourceFile_TB/additional_params.csv
diff --git a/resources/ResourceFile_TB/all_districts.csv b/resources/ResourceFile_TB/all_districts.csv
diff --git a/resources/ResourceFile_TB/calibrated_transmission_rates.csv b/resources/ResourceFile_TB/calibrated_transmission_rates.csv
diff --git a/resources/ResourceFile_TB/cases2010district.csv b/resources/ResourceFile_TB/cases2010district.csv
diff --git a/resources/ResourceFile_TB/details_rates.csv b/resources/ResourceFile_TB/details_rates.csv
diff --git a/resources/ResourceFile_TB/followup.csv b/resources/ResourceFile_TB/followup.csv
diff --git a/resources/ResourceFile_TB/ipt_coverage.csv b/resources/ResourceFile_TB/ipt_coverage.csv
diff --git a/resources/ResourceFile_TB/latent_TB2014_summary.csv b/resources/ResourceFile_TB/latent_TB2014_summary.csv
diff --git a/resources/ResourceFile_TB/parameters.csv b/resources/ResourceFile_TB/parameters.csv
diff --git a/resources/ResourceFile_TB/pulm_tb.csv b/resources/ResourceFile_TB/pulm_tb.csv
diff --git a/resources/ResourceFile_TB/scaleup_parameters.csv b/resources/ResourceFile_TB/scaleup_parameters.csv
diff --git a/resources/ResourceFile_TB/testing_rates.csv b/resources/ResourceFile_TB/testing_rates.csv
diff --git a/resources/malaria/ResourceFile_malaria.xlsx b/resources/malaria/ResourceFile_malaria.xlsx
diff --git a/resources/malaria/ResourceFile_malaria/MAP_CommoditiesData2023.csv b/resources/malaria/ResourceFile_malaria/MAP_CommoditiesData2023.csv
diff --git a/resources/malaria/ResourceFile_malaria/MAP_IRSrates.csv b/resources/malaria/ResourceFile_malaria/MAP_IRSrates.csv
diff --git a/resources/malaria/ResourceFile_malaria/MAP_ITNrates.csv b/resources/malaria/ResourceFile_malaria/MAP_ITNrates.csv
diff --git a/resources/malaria/ResourceFile_malaria/MAP_InfectionData2023.csv b/resources/malaria/ResourceFile_malaria/MAP_InfectionData2023.csv
diff --git a/resources/malaria/ResourceFile_malaria/NMCP.csv b/resources/malaria/ResourceFile_malaria/NMCP.csv
diff --git a/resources/malaria/ResourceFile_malaria/PfPR_MAPdata.csv b/resources/malaria/ResourceFile_malaria/PfPR_MAPdata.csv
diff --git a/resources/malaria/ResourceFile_malaria/WHO_CaseData2023.csv b/resources/malaria/ResourceFile_malaria/WHO_CaseData2023.csv
diff --git a/resources/malaria/ResourceFile_malaria/WHO_MalReport.csv b/resources/malaria/ResourceFile_malaria/WHO_MalReport.csv
diff --git a/resources/malaria/ResourceFile_malaria/WHO_TestData2023.csv b/resources/malaria/ResourceFile_malaria/WHO_TestData2023.csv
diff --git a/resources/malaria/ResourceFile_malaria/WHOcommodities.csv b/resources/malaria/ResourceFile_malaria/WHOcommodities.csv
diff --git a/resources/malaria/ResourceFile_malaria/highrisk_districts.csv b/resources/malaria/ResourceFile_malaria/highrisk_districts.csv
diff --git a/resources/malaria/ResourceFile_malaria/inc1000py_MAPdata.csv b/resources/malaria/ResourceFile_malaria/inc1000py_MAPdata.csv
diff --git a/resources/malaria/ResourceFile_malaria/interventions.csv b/resources/malaria/ResourceFile_malaria/interventions.csv
diff --git a/resources/malaria/ResourceFile_malaria/mortalityRate_MAPdata.csv b/resources/malaria/ResourceFile_malaria/mortalityRate_MAPdata.csv
diff --git a/resources/malaria/ResourceFile_malaria/parameters.csv b/resources/malaria/ResourceFile_malaria/parameters.csv
diff --git a/resources/malaria/ResourceFile_malaria/scaleup_parameters.csv b/resources/malaria/ResourceFile_malaria/scaleup_parameters.csv
diff --git a/resources/malaria/ResourceFile_malaria/severe_symptoms.csv b/resources/malaria/ResourceFile_malaria/severe_symptoms.csv
diff --git a/resources/malaria/ResourceFile_malaria/txCov_MAPdata.csv b/resources/malaria/ResourceFile_malaria/txCov_MAPdata.csv
diff --git a/src/tlo/methods/epi.py b/src/tlo/methods/epi.py
@@ -7,6 +7,7 @@
 from tlo.events import IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
 from tlo.methods import Metadata
 from tlo.methods.hsi_event import HSI_Event
+from tlo.util import read_csv_files
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -71,9 +72,7 @@ def __init__(self, name=None, resourcefilepath=None):
 
     def read_parameters(self, data_folder):
         p = self.parameters
-        workbook = pd.read_excel(
-            Path(self.resourcefilepath) / 'ResourceFile_EPI_WHO_estimates.xlsx', sheet_name=None
-        )
+        workbook = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_EPI_WHO_estimates', files=None)
 
         self.load_parameters_from_dataframe(workbook["parameters"])
 

diff --git a/src/tlo/methods/hiv.py b/src/tlo/methods/hiv.py
@@ -25,7 +25,6 @@
 """
 from __future__ import annotations
 
-import os
 from typing import TYPE_CHECKING, List
 
 import numpy as np
@@ -40,7 +39,7 @@
 from tlo.methods.hsi_event import HSI_Event
 from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
 from tlo.methods.symptommanager import Symptom
-from tlo.util import create_age_range_lookup
+from tlo.util import create_age_range_lookup, read_csv_files
 
 if TYPE_CHECKING:
     from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
@@ -423,10 +422,7 @@ def read_parameters(self, data_folder):
         # Shortcut to parameters dict
         p = self.parameters
 
-        workbook = pd.read_excel(
-            os.path.join(self.resourcefilepath, "ResourceFile_HIV.xlsx"),
-            sheet_name=None,
-        )
+        workbook = read_csv_files(self.resourcefilepath/'ResourceFile_HIV', files=None)
         self.load_parameters_from_dataframe(workbook["parameters"])
 
         # Load data on HIV prevalence

diff --git a/src/tlo/methods/malaria.py b/src/tlo/methods/malaria.py
@@ -20,7 +20,7 @@
 from tlo.methods.hsi_event import HSI_Event
 from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
 from tlo.methods.symptommanager import Symptom
-from tlo.util import random_date
+from tlo.util import random_date, read_csv_files
 
 if TYPE_CHECKING:
     from tlo.methods.hsi_generic_first_appts import DiagnosisFunction, HSIEventScheduler
@@ -228,7 +228,8 @@ def __init__(self, name=None, resourcefilepath=None):
     }
 
     def read_parameters(self, data_folder):
-        workbook = pd.read_excel(self.resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx', sheet_name=None)
+        # workbook = pd.read_excel(self.resourcefilepath / 'malaria' / 'ResourceFile_malaria.xlsx', sheet_name=None)
+        workbook = read_csv_files(self.resourcefilepath / 'malaria' / 'ResourceFile_malaria', files=None)
         self.load_parameters_from_dataframe(workbook['parameters'])
 
         p = self.parameters

diff --git a/src/tlo/methods/measles.py b/src/tlo/methods/measles.py
@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import math
-import os
 from typing import TYPE_CHECKING, List
 
 import pandas as pd
@@ -13,7 +12,7 @@
 from tlo.methods.hsi_event import HSI_Event
 from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
 from tlo.methods.symptommanager import Symptom
-from tlo.util import random_date
+from tlo.util import random_date, read_csv_files
 
 if TYPE_CHECKING:
     from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
@@ -102,11 +101,7 @@ def __init__(self, name=None, resourcefilepath=None):
     def read_parameters(self, data_folder):
         """Read parameter values from file
         """
-
-        workbook = pd.read_excel(
-            os.path.join(self.resourcefilepath, "ResourceFile_Measles.xlsx"),
-            sheet_name=None,
-        )
+        workbook = read_csv_files(self.resourcefilepath/'ResourceFile_Measles', files=None)
         self.load_parameters_from_dataframe(workbook["parameters"])
 
         self.parameters["symptom_prob"] = workbook["symptoms"]