Merge pull request #20 from OpenIsraeliSupermarkets/patch

v0.1.5

Author: erlichsefi

il_supermarket_parsers/parsers/hazi_hinam.py

@@ -1,5 +1,8 @@
 from il_supermarket_parsers.engines import BaseFileConverter
-from il_supermarket_parsers.documents import XmlDataFrameConverter
+from il_supermarket_parsers.documents import (
+    XmlDataFrameConverter,
+    SubRootedXmlDataFrameConverter,
+)
 
 
 class HaziHinamFileConverter(BaseFileConverter):
@@ -33,10 +36,12 @@ def __init__(self) -> None:
                 roots=["ChainId", "SubChainId", "StoreId", "BikoretNo"],
                 ignore_column=["XmlDocVersion", "DllVerNo"],
             ),
-            stores_parser=XmlDataFrameConverter(
+            stores_parser=SubRootedXmlDataFrameConverter(
                 id_field="StoreID",
-                list_key="Stores",
-                roots=["ChainId"],
+                list_key="SubChains",
+                roots=["ChainId", "ChainName", "LastUpdateDate", "LastUpdateTime"],
+                list_sub_key="Stores",
+                sub_roots=["SubChainID", "SubChainName"],
                 ignore_column=[],
             ),
         )

il_supermarket_parsers/parsers/other.py

@@ -1,4 +1,6 @@
 from il_supermarket_parsers.engines.base import BaseFileConverter
+from il_supermarket_parsers.documents import SubRootedXmlDataFrameConverter
+
 from .confix import CofixFileConverter
 
 
@@ -153,3 +155,26 @@ class WoltFileConverter(BaseFileConverter):
     """
     wolt
     """
+
+    def __init__(self) -> None:
+        super().__init__(
+            promofull_parser=SubRootedXmlDataFrameConverter(
+                list_key="Promotions",
+                id_field="PromotionId",
+                roots=["ChainId", "SubChainId", "StoreId", "BikoretNo"],
+                list_sub_key="PromotionItems",
+                sub_roots=[
+                    "Remarks",
+                    "AdditionalRestrictions",
+                    "ClubId",
+                    "PromotionEndHour",
+                    "PromotionUpdateTime",
+                    "PromotionId",
+                    "PromotionDescription",
+                    "PromotionStartDate",
+                    "PromotionStartHour",
+                    "PromotionEndDate",
+                ],
+                ignore_column=["XmlDocVersion", "DllVerNo"],
+            )
+        )

il_supermarket_parsers/parsers/tests/test_case.py

@@ -75,6 +75,12 @@ def __parser_validate(self, file_type, dump_path="temp"):
                 files_types=[file_type],
             ).load()
 
+            complete_file_loaded = list(map(lambda x: x.get_full_path(), files))
+            files_from_folder = self.list_xml_files_recursive(sub_folder)
+            assert sorted(complete_file_loaded) == sorted(files_from_folder), (
+                f"dataloader failed, failed to load"
+                f": {list(set(files_from_folder) - set(complete_file_loaded))}"
+            )
             dfs = []
             for file in files:
 
@@ -97,6 +103,15 @@ def __parser_validate(self, file_type, dump_path="temp"):
             if dfs:
                 pd.concat(dfs)
 
+        def list_xml_files_recursive(self, directory):
+            """list all xml files"""
+            file_list = []
+            for root, _, files in os.walk(directory):
+                for file in files:
+                    if "xml" in file:
+                        file_list.append(os.path.join(root, file))
+            return file_list
+
         def test_parsing_store(self):
             """scrape one file and make sure it exists"""
             self._parser_validate(FileTypesFilters.STORE_FILE.name)

il_supermarket_parsers/utils/data_loader.py

@@ -66,13 +66,20 @@ def _format_datetime(self, date):
 
     def _file_name_to_components(self, store_folder, file_name, empty_store_id="0000"):
         """extract file name components"""
+
+        _file_name_split = file_name.split(".")[0].split("-")
         try:
-            prefix_file_name, store_number, date, *_ = file_name.split(".")[0].split(
-                "-"
-            )
+            # Promo7290700100008-000-207-20250224-103225
+            if len(_file_name_split) == 5:
+                prefix_file_name, _, store_number, date, time, *_ = _file_name_split
+                extracted_datetime = date + time
+            else:
+                prefix_file_name, store_number, extracted_datetime, *_ = (
+                    _file_name_split
+                )
         except ValueError:
             # global files
-            prefix_file_name, date, *_ = file_name.split(".")[0].split("-")
+            prefix_file_name, extracted_datetime, *_ = _file_name_split
             store_number = empty_store_id
 
         file_type, chain_id = self._find_file_type_and_chain_id(prefix_file_name)
@@ -83,7 +90,7 @@ def _file_name_to_components(self, store_folder, file_name, empty_store_id="0000
             prefix_file_name=prefix_file_name,
             extracted_store_number=store_number,
             extracted_chain_id=chain_id,
-            extracted_date=self._format_datetime(date),
+            extracted_date=self._format_datetime(extracted_datetime),
             detected_filetype=file_type,
         )
 

il_supermarket_parsers/utils/xml_utils.py

@@ -1,4 +1,5 @@
 import xml.etree.ElementTree as ET
+from lxml import etree
 
 
 def count_tag_in_xml(xml_file_path, tag_to_count):
@@ -83,13 +84,31 @@ def change_xml_encoding(file_path):
         )
 
 
+def try_to_recover_xml(file_path):
+    """try to recover the xml"""
+
+    parser = etree.XMLParser(recover=True, encoding="utf-8")
+    with open(file_path, "rb") as f:
+        tree = etree.parse(f, parser)
+    fixed_xml = etree.tostring(tree, pretty_print=True, encoding="utf-8").decode(
+        "utf-8"
+    )
+
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(fixed_xml)
+
+
 def get_root(file):
     """get ET root"""
     try:
         tree = ET.parse(file)
     except ET.ParseError:
-        change_xml_encoding(file)
-        tree = ET.parse(file)
+        try:
+            try_to_recover_xml(file)
+            tree = ET.parse(file)
+        except ET.ParseError:
+            change_xml_encoding(file)
+            tree = ET.parse(file)
 
     return tree.getroot()
 

requirements.txt

@@ -1,5 +1,5 @@
 pandas==2.2.2
 lxml==5.2.1
 pymongo==4.6.3
-il-supermarket-scraper>=0.5.6
+il-supermarket-scraper>=0.5.7
 tqdm==4.66

setup.py

@@ -31,7 +31,7 @@
     tests_require=dev_required,
     extras_require={"test": ["pytest", "pytest-xdist"]},
     # *strongly* suggested for sharing
-    version="0.1.4",
+    version="0.1.5",
     # The license can be anything you like
     license="MIT",
     description="python package that process the data dumped by the israeli supermarket",