Convert excel files to csv files. Joe's modules (#1508)

* convert excel files to csv equivalent - Joe modules

* use new method to read newly converted csv files

* sort imports

* used read_csv_files method

* read csv files using the new read_csv_files method, disable pylint error, update doc string

* edits

* convert Excel to csv file(s), change from read_excel to read_csv_files for newly created csv file(s)

* rename resource files. added underscores and removed some unwanted characters and duplicate extensions

* updated modules to align with updates to read_csv_files method in master

* remove disable pylint as we are now directly reading via pd.read_csv

* remove unused import

* delete unused csv files

---------

Co-authored-by: joehcollins <[email protected]>

Author: mnjowe

resources/ResourceFile_AntenatalCare.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5cf6bb4312c4e315d004ef769bac35560be3fe423fe648e9ea5549b5e3342d1
+size 1258

resources/ResourceFile_AntenatalCare/parameter_values.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be621298177fff272051d8b519dd890f812c427603f24e70a66386e528157047
+size 11243

resources/ResourceFile_LabourSkilledBirthAttendance.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f805646e16fe9912a68da5a10b901086e8a531f4802b04a7ec2b1216eedfac4e
+size 3737

resources/ResourceFile_LabourSkilledBirthAttendance/parameter_values.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dac89c4cc638a6f65cc21effb521380e3d6ca06600af737244e41ef0f71b4a9
+size 3573

resources/ResourceFile_NewbornOutcomes.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:702d26f19582a9fc603265da9f0b61f89fede8aa3582ab8cee551cf4109a49b1
+size 12966

resources/ResourceFile_NewbornOutcomes/parameter_values.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3b8d2085d10f1680cf7d521ba415777b194ef26a964c14bca473cbdd76c7982
+size 770

resources/ResourceFile_PostnatalSupervisor.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ab5f87adb1c35d4ff2fa0e6226a32840ef4d3d4a8d130404cf74e9f2a13528a
+size 296

resources/ResourceFile_PostnatalSupervisor/parameter_values.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d54deeb3a78189d18f1e18e20132aa4e965d5e780ab6299e4ef5434627449dfe
+size 259

resources/ResourceFile_PregnancySupervisor.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dec00bfe6c52595dbac43cc05bfa00771408dd3ceddc2a0c843ffe232453fd04
+size 270

resources/ResourceFile_PregnancySupervisor/parameter_values.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb59b96ca917f218d48b349790092ea04d612c3545669c219ae25a65930a38d1
+size 810

resources/contraception/ResourceFile_Contraception.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62709e4159aff94254a843ad726cfb0b5d655bce4c7d209cff03fed2adde3534
+size 268

resources/contraception/ResourceFile_Contraception/Discontinuation_ByAge.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66624dbc15b3f8299e218c97e1f9ebd2e79fc382e25b55f214cdfbe50cd85ee3
+size 150

resources/contraception/ResourceFile_Contraception/Discontinuation_ByMethod.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66624dbc15b3f8299e218c97e1f9ebd2e79fc382e25b55f214cdfbe50cd85ee3
+size 150

resources/contraception/ResourceFile_Contraception/Failure_ByMethod.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fafc2deddb6c9f1a341f0dc45097aaf0770cb09b49ec26ffa7133f35e748044
+size 5983

resources/contraception/ResourceFile_Contraception/Initiation_AfterBirth.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6848831b52ab636cf78cd6c3ddc999ad01a46d9d5471987d0e406c479f75ef0
+size 263

resources/contraception/ResourceFile_Contraception/Initiation_ByAge.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57d060bdd1a4ad87630853291de818144eeb15c5d2a5ba54a4d2507b31f879ff
+size 509

resources/contraception/ResourceFile_Contraception/Initiation_ByMethod.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9340fdca9143fd824c0074986c0fc4c7620120517ecbf67bce75b32d4fa240d
+size 295

resources/contraception/ResourceFile_Contraception/Interventions_PPFP.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3784438c4ccc28e3886cc8244880364a3b73581598e2b73e515f2309fd14a281
+size 1219

resources/contraception/ResourceFile_Contraception/Interventions_Pop.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71c1d00707d77dcba8ec87532a9ac3f122cb4bbb2c6cd473d57c0157259b16e3
+size 95

resources/contraception/ResourceFile_Contraception/Method_Use_In_2010.csv

@@ -1,7 +1,7 @@
 """
 A helper script to see the numbers of women of reproductive age having female sterilisation per 5-years age categories +
 total, and the number of all women in the population in 2010 and 2020, to help to calibrate the intervention multipliers
-(saved in ResourceFile_Contraception.xlsx in the sheets Interventions_Pop & Interventions_PPFP).
+(saved in ResourceFile_Contraception folder in the Interventions_Pop & Interventions_PPFP CSV files).
 """
 from pathlib import Path
 

resources/contraception/ResourceFile_Contraception/Pregnancy_NotUsing_HIVeffect.csv

@@ -12,6 +12,7 @@
 from tlo.methods.labour import LabourOnsetEvent
 from tlo.methods.malaria import HSI_MalariaIPTp
 from tlo.methods.tb import HSI_Tb_ScreeningAndRefer
+from tlo.util import read_csv_files
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -176,8 +177,8 @@ def __init__(self, name=None, resourcefilepath=None):
     }
 
     def read_parameters(self, data_folder):
-        parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_AntenatalCare.xlsx',
-                                            sheet_name='parameter_values')
+        parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_AntenatalCare',
+                                            files='parameter_values')
         self.load_parameters_from_dataframe(parameter_dataframe)
 
     def initialise_population(self, population):

resources/contraception/ResourceFile_Contraception/Pregnancy_NotUsing_In_2010.csv

@@ -8,7 +8,7 @@
 from tlo.analysis.utils import flatten_multi_index_series_into_dict_for_logging
 from tlo.events import Event, IndividualScopeEventMixin, PopulationScopeEventMixin, RegularEvent
 from tlo.methods.hsi_event import HSI_Event
-from tlo.util import random_date, sample_outcome, transition_states
+from tlo.util import random_date, read_csv_files, sample_outcome, transition_states
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -164,7 +164,8 @@ def read_parameters(self, data_folder):
         """Import the relevant sheets from the ResourceFile (excel workbook) and declare values for other parameters
         (CSV ResourceFile).
         """
-        workbook = pd.read_excel(Path(self.resourcefilepath) / 'contraception' / 'ResourceFile_Contraception.xlsx', sheet_name=None)
+        workbook = read_csv_files(Path(self.resourcefilepath) / 'contraception' / 'ResourceFile_Contraception',
+                                  files=None)
 
         # Import selected sheets from the workbook as the parameters
         sheet_names = [
@@ -1350,10 +1351,10 @@ def __init__(self, *args):
         super().__init__(name='Labour')
 
     def read_parameters(self, *args):
-        parameter_dataframe = pd.read_excel(self.sim.modules['Contraception'].resourcefilepath /
+        parameter_dataframe = read_csv_files(self.sim.modules['Contraception'].resourcefilepath /
                                             'contraception' /
-                                            'ResourceFile_Contraception.xlsx',
-                                            sheet_name='simplified_labour_parameters')
+                                            'ResourceFile_Contraception',
+                                            files='simplified_labour_parameters')
         self.load_parameters_from_dataframe(parameter_dataframe)
 
     def initialise_population(self, population):

resources/contraception/ResourceFile_Contraception/Prob_Switch_From.csv

@@ -17,7 +17,7 @@
 from tlo.methods.hsi_event import HSI_Event
 from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
 from tlo.methods.postnatal_supervisor import PostnatalWeekOneMaternalEvent
-from tlo.util import BitsetHandler
+from tlo.util import BitsetHandler, read_csv_files
 
 if TYPE_CHECKING:
     from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
@@ -620,9 +620,9 @@ def __init__(self, name=None, resourcefilepath=None):
     }
 
     def read_parameters(self, data_folder):
-        parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_LabourSkilledBirth'
-                                                                          'Attendance.xlsx',
-                                            sheet_name='parameter_values')
+        parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_LabourSkilledBirth'
+                                                                          'Attendance',
+                                            files='parameter_values')
         self.load_parameters_from_dataframe(parameter_dataframe)
 
     def initialise_population(self, population):

resources/contraception/ResourceFile_Contraception/Prob_Switch_From_And_To.csv

@@ -10,7 +10,7 @@
 from tlo.methods.causes import Cause
 from tlo.methods.hsi_event import HSI_Event
 from tlo.methods.postnatal_supervisor import PostnatalWeekOneNeonatalEvent
-from tlo.util import BitsetHandler
+from tlo.util import BitsetHandler, read_csv_files
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -311,8 +311,8 @@ def __init__(self, name=None, resourcefilepath=None):
 
     def read_parameters(self, data_folder):
 
-        parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_NewbornOutcomes.xlsx',
-                                            sheet_name='parameter_values')
+        parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_NewbornOutcomes',
+                                            files='parameter_values')
         self.load_parameters_from_dataframe(parameter_dataframe)
 
         # Here we map 'disability' parameters to associated DALY weights to be passed to the health burden module

resources/contraception/ResourceFile_Contraception/simplified_labour_parameters.csv

@@ -9,6 +9,7 @@
 from tlo.methods import Metadata, postnatal_supervisor_lm, pregnancy_helper_functions
 from tlo.methods.causes import Cause
 from tlo.methods.hsi_event import HSI_Event
+from tlo.util import read_csv_files
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -230,8 +231,8 @@ def __init__(self, name=None, resourcefilepath=None):
     }
 
     def read_parameters(self, data_folder):
-        parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PostnatalSupervisor.xlsx',
-                                            sheet_name='parameter_values')
+        parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_PostnatalSupervisor',
+                                            files='parameter_values')
         self.load_parameters_from_dataframe(parameter_dataframe)
 
     def initialise_population(self, population):

src/scripts/contraception/f_steril_use_2010vs2020.py

@@ -27,7 +27,7 @@
 )
 from tlo.methods.causes import Cause
 from tlo.methods.hsi_generic_first_appts import GenericFirstAppointmentsMixin
-from tlo.util import BitsetHandler
+from tlo.util import BitsetHandler, read_csv_files
 
 if TYPE_CHECKING:
     from tlo.methods.hsi_generic_first_appts import HSIEventScheduler
@@ -434,8 +434,8 @@ def __init__(self, name=None, resourcefilepath=None):
 
     def read_parameters(self, data_folder):
         # load parameters from the resource file
-        parameter_dataframe = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_PregnancySupervisor.xlsx',
-                                            sheet_name='parameter_values')
+        parameter_dataframe = read_csv_files(Path(self.resourcefilepath) / 'ResourceFile_PregnancySupervisor',
+                                            files='parameter_values')
         self.load_parameters_from_dataframe(parameter_dataframe)
 
         # Here we map 'disability' parameters to associated DALY weights to be passed to the health burden module.

src/tlo/methods/care_of_women_during_pregnancy.py

@@ -111,7 +111,7 @@ def read_parameters(self, data_folder):
         self.parameters['months_between_pregnancy_and_delivery'] = 9
 
         # Breastfeeding status for newborns (importing from the Newborn resourcefile)
-        rf = pd.read_excel(Path(self.resourcefilepath) / 'ResourceFile_NewbornOutcomes.xlsx')
+        rf = pd.read_csv(Path(self.resourcefilepath) / 'ResourceFile_NewbornOutcomes/parameter_values.csv')
         param_as_string = rf.loc[rf.parameter_name == 'prob_breastfeeding_type']['value'].iloc[0]
         parameter = json.loads(param_as_string)[0]
         self.parameters['prob_breastfeeding_type'] = parameter

src/tlo/methods/contraception.py

@@ -12,6 +12,7 @@
 from tlo.methods import contraception, demography, enhanced_lifestyle, healthsystem, symptommanager
 from tlo.methods.contraception import HSI_Contraception_FamilyPlanningAppt
 from tlo.methods.hiv import DummyHivModule
+from tlo.util import read_csv_files
 
 
 def run_sim(tmpdir,
@@ -918,8 +919,8 @@ def test_input_probs_sum():
 
     # Import relevant sheets from the workbook
     resourcefilepath = Path(os.path.dirname(__file__)) / '../resources'
-    workbook = pd.read_excel(Path(resourcefilepath) / 'contraception' / 'ResourceFile_Contraception.xlsx',
-                             sheet_name=None)
+    workbook = read_csv_files(Path(resourcefilepath) / 'contraception' / 'ResourceFile_Contraception',
+                             files=None)
     sheet_names = [
         'Initiation_ByMethod',
         'Interventions_Pop',