next version of scrapers

Author: erlichsefi

il_supermarket_parsers/raw_parsing_pipeline.py

@@ -1,7 +1,8 @@
 import os
+from typing import List
 from tqdm import tqdm
 from .parser_factory import ParserFactory
-from .utils import DataLoader
+from .utils import DataLoader, DumpFile
 
 
 class RawParsingPipeline:
@@ -23,7 +24,7 @@ def process(self, limit=None):
             self.file_type.lower() + "_" + self.store_name.lower() + ".csv",
         )
 
-        files_to_process = DataLoader(
+        files_to_process: List[DumpFile] = DataLoader(
             self.folder,
             store_names=[self.store_name],
             files_types=[self.file_type],
@@ -34,18 +35,38 @@ def process(self, limit=None):
             total=len(files_to_process),
             desc=f"Processing {self.file_type}@{self.store_name}",
         ):
-            parser = parser_class()
-            df = parser.read(file)
 
-            if not os.path.exists(create_csv):
-                df.to_csv(create_csv, index=False, mode="w", header=True)
-            else:
-                df.to_csv(create_csv, index=False, mode="a", header=False)
-
-            del df
+            execution_log = []
+            try:
+                parser = parser_class()
+                df = parser.read(file)
+
+                if not os.path.exists(create_csv):
+                    df.to_csv(create_csv, index=False, mode="w", header=True)
+                else:
+                    df.to_csv(create_csv, index=False, mode="a", header=False)
+
+                del df
+
+                execution_log.append(
+                    {
+                        "status": True,
+                        **file.to_log_dict(),
+                    }
+                )
+
+            except Exception as error:  # pylint: disable=broad-exception-caught
+                execution_log.append(
+                    {
+                        "status": False,
+                        "error": error,
+                        **file.to_log_dict(),
+                    }
+                )
 
         return {
             "status": True,
+            "execution_log": execution_log,
             "file_was_created": len(files_to_process) > 0,
             "file_created_path": create_csv,
             "files_to_process": [dumpfile.file_name for dumpfile in files_to_process],

il_supermarket_parsers/utils/data_loader.py

@@ -25,6 +25,18 @@ def get_full_path(self):
         """get full file path"""
         return os.path.join(self.store_folder, self.file_name)
 
+    def to_log_dict(self):
+        """return the object as dict"""
+        return {
+            "store_folder": self.store_folder,
+            "file_name": self.file_name,
+            "prefix_file_name": self.prefix_file_name,
+            "extracted_store_number": self.extracted_store_number,
+            "extracted_chain_id": self.extracted_chain_id,
+            "extracted_date": self.extracted_date.strftime("%Y-%m-%d %H:%M:%S"),
+            "detected_filetype": self.detected_filetype.name,
+        }
+
 
 class DataLoader:
     """class for loading dump files from the folder"""

requirements.txt

@@ -1,5 +1,5 @@
 pandas==2.2.2
 lxml==5.2.1
 pymongo==4.6.3
-il-supermarket-scraper>=0.5.0
+il-supermarket-scraper>=0.5.1
 tqdm==4.66