Merge pull request #6 from kubantjan/test-pr

Improve tests, fix a little bit

Author: kubantjan

conda.yml

@@ -2,5 +2,5 @@ name: fast-form
 channels:
   - conda-forge
 dependencies:
-  - python=3.7.9
+  - python=3.7.10
   - poppler=20.09.0

fast_form/outputting/process_document.py

@@ -1,5 +1,6 @@
 import logging
 import os
+from typing import Optional
 
 from fast_form.config.configuration_dataclasses import ProcessingConfig
 from fast_form.field_recognizer.recognize_all import recognize
@@ -13,11 +14,12 @@
 
 
 def process_document_and_add_to_validation_excel(document_path: str, processing_config: ProcessingConfig,
-                                                 excel_path: str):
+                                                 excel_path: str, patient_id: Optional[str] = None):
     if os.path.exists(document_path):
-        maybe_patient_id = os.path.basename(document_path).split(".")[0]
+        if not patient_id:
+            patient_id = os.path.basename(document_path).split(".")[0]
         form_data = process_document(processing_config, document_path=document_path)
-        df, images = output_data(form_data, maybe_patient_id=maybe_patient_id)
+        df, images = output_data(form_data, maybe_patient_id=patient_id)
 
         save_data(df, images, excel_path=excel_path)
     else:

fast_form/outputting/utils_for_main.py

@@ -3,12 +3,14 @@
 import os
 
 import dacite
+import openpyxl
 import pandas as pd
 from openpyxl import load_workbook
 
 from fast_form.config.configuration_dataclasses import PathsForProcessingConfig
 from fast_form.config.configuration_loading import get_processing_config
 from fast_form.outputting.process_document import process_document_and_add_to_validation_excel
+from fast_form.structure_parser.form_structure_dataclasses import FieldType
 
 SHEET_WITH_RESULTS = "automatic_results"
 VALIDATION_EXCEL_NAME = "validation_excel.xlsx"
@@ -25,7 +27,7 @@ def process_to_validation_excel(paths_for_processing_config: PathsForProcessingC
     document_names = [file for file in os.listdir(paths_for_processing_config.folder_with_documents_path) if
                       file.endswith('.pdf') or file.endswith(".jpg") or file.endswith(".png")]
     logger.info(f"Processing to validation excel from {paths_for_processing_config.folder_with_documents_path}. Number"
-                 f" of documents is {len(document_names)}")
+                f" of documents is {len(document_names)}")
     for document_name in document_names:
         logging.debug(f"Processing document {document_name}")
         process_document_and_add_to_validation_excel(
@@ -48,6 +50,7 @@ def process_to_final_excel(paths_for_processing_config: PathsForProcessingConfig
     one_patient_per_row_df = (
         validation_df
             .set_index(['patient_id', 'name'])
+            .loc[lambda df: df.field_type == FieldType.SINGLE_CHOICE]
             .data
             .apply(lambda response_or_error: response_or_error if response_or_error >= 0 else "")
             .unstack(level=1)
@@ -65,8 +68,10 @@ def process_to_final_excel(paths_for_processing_config: PathsForProcessingConfig
                 sheet_name=SHEET_WITH_RESULTS)
 
             combined_one_patient_per_row_df = one_patient_per_row_df.combine_first(old_one_patient_per_row_df)
+        book = load_workbook(paths_for_processing_config.final_excel_path)
+    else:
+        book = openpyxl.Workbook()
 
-    book = load_workbook(paths_for_processing_config.final_excel_path)
     writer = pd.ExcelWriter(paths_for_processing_config.final_excel_path, engine='openpyxl')
     if SHEET_WITH_RESULTS in book.sheetnames:
         book.remove(book[SHEET_WITH_RESULTS])

fast_form/tests/test_basic_questionnaire.py

@@ -6,7 +6,8 @@
 from fast_form.config.configuration_loading import get_processing_config
 from fast_form.outputting.ouput_data import output_data
 from fast_form.outputting.process_document import process_document, process_document_and_add_to_validation_excel
-from fast_form.outputting.utils_for_main import load_paths_for_processing_config, VALIDATION_EXCEL_NAME
+from fast_form.outputting.utils_for_main import load_paths_for_processing_config, VALIDATION_EXCEL_NAME, \
+    process_to_final_excel
 
 
 class TestWholeProcess(unittest.TestCase):
@@ -35,7 +36,10 @@ def test_process_to_excel(self):
             os.remove(validation_excel_path)
         process_document_and_add_to_validation_excel(document_path=self.document_path,
                                                      processing_config=self.processing_config,
-                                                     excel_path=validation_excel_path)
+                                                     excel_path=validation_excel_path,
+                                                     patient_id="test1")
         process_document_and_add_to_validation_excel(document_path=self.document_path,
                                                      processing_config=self.processing_config,
-                                                     excel_path=validation_excel_path)
+                                                     excel_path=validation_excel_path,
+                                                     patient_id="test2")
+        process_to_final_excel(self.paths_for_processing_config)

fast_form/tests/test_special_questionnaire.py

@@ -3,7 +3,8 @@
 
 from fast_form.config.configuration_loading import get_processing_config
 from fast_form.outputting.process_document import process_document_and_add_to_validation_excel
-from fast_form.outputting.utils_for_main import load_paths_for_processing_config, VALIDATION_EXCEL_NAME
+from fast_form.outputting.utils_for_main import load_paths_for_processing_config, VALIDATION_EXCEL_NAME, \
+    process_to_final_excel
 
 
 class TestWholeProcess(unittest.TestCase):
@@ -21,3 +22,4 @@ def test_process_to_excel(self):
         process_document_and_add_to_validation_excel(document_path=self.document_path,
                                                      processing_config=self.processing_config,
                                                      excel_path=self.validation_excel_path)
+        process_to_final_excel(self.paths_for_processing_config)

setup.py

@@ -81,10 +81,11 @@
     # requirements files see:
     # https://packaging.python.org/en/latest/requirements.html
     install_requires=[
+        'scipy<=1.4.1'
         'numpy>=1.18.3',
         'opencv-contrib-python>=4.4.0.42',
-        'pandas>=1.1.4',
         'tensorflow<=2.1.0',
+        'pandas>=1.1.4',
         'keras<=2.3.1',
         'autopep8>=1.5.4',
         'dacite>=1.5.1',