add the html_dynamic parse mode

Author: BertramYe

.gitignore

@@ -1,3 +1,4 @@
+__pycache__
 *__pycache__
 Pipfile
 result

InstallHelper.py

@@ -0,0 +1,132 @@
+from os import path,getenv,listdir
+from setuptools import Command
+from subprocess import check_call
+from setuptools.command.install import install
+from typing import Literal,Optional
+import sys
+from platform import system
+
+
+class BrowserCoreInstaller(Command):
+    """
+        install the Browser Core
+    """
+    # description = "Install the specified preparser browser core (chromium, firefox, or webkit)"
+    # define command
+    # user_options = [
+    #     ('browser=', None, 'Specify the preparser browser to install: chromium, firefox, or webkit')
+    # ]
+
+    def initialize_options(self):
+        self.browser_list = ['chromium','firefox','webkit']
+        self.installed_browsers = []
+        self.need_recheck_browsers = []
+        self.browser_cache_path = None
+        self.preched_installed_browsers = self._get_pre_installed_browsers()
+
+    def _get_pre_installed_browsers(self) -> list[str]:
+        preched_installed_browsers = []
+        os_type = system()
+        browser_cache_path = path.expanduser("~/.cache/ms-playwright/") 
+        if os_type == "Windows":
+            browser_cache_path = path.join(getenv("APPDATA"), "Local", "ms-playwright")  
+        if path.exists(browser_cache_path):
+            self.browser_cache_path = browser_cache_path
+            folder_name_list = listdir(self.browser_cache_path)
+            for folder_name in folder_name_list:
+                abs_path = path.join(self.browser_cache_path, folder_name)
+                if path.isdir(abs_path):
+                    for key in self.browser_list:
+                        if folder_name.startswith(key):
+                            preched_installed_browsers.append(key)
+        else:
+            preched_installed_browsers = []
+        return preched_installed_browsers
+    
+
+    def precheck_installed_browsers(self):
+        self.installed_browsers = []
+        self.need_recheck_browsers = []
+        # get the path
+        for browser_name in self.browser_list:
+            if browser_name in self.preched_installed_browsers:
+                print(f'find that browser {browser_name} of preparser installed !')
+                operate_choice = self.check_choice_avalible(f" do you want to reinstall,remove or keep it ? (1 : reinstall, 2: remove , 3: keep.): ",['1','2','3'])
+                if operate_choice == "3":
+                    self.installed_browsers.append(browser_name)
+                else:  # "1" or "2"
+                    self.operate_browser("uninstall",browser_name)
+                    if operate_choice == '1':
+                        self.operate_browser("install",browser_name)
+                        self.installed_browsers.append(browser_name)
+            else:
+                self.need_recheck_browsers.append(browser_name)
+        
+
+    def init_install_browser(self):
+        # if not , just let the user to choose
+        print("please choose a preparser browser  to install: ")
+        print("[1] chromium, [2] firefox, [3] webkit.")
+        choice = self.check_choice_avalible(f'please input a number to choose a browser (1/2/3):',['1','2','3'])
+        browser = self.browser_list[int(choice)-1]
+        self.operate_browser("install",browser)
+        return browser
+    
+    def operate_browser(self,command_type:Literal["install","uninstall"], browser_name:str):
+        # install specified browser
+        print(f"{command_type}ing preparser browser {browser_name} ...")
+        check_call([sys.executable, "-m", "playwright", command_type, browser_name])    
+
+    def check_choice_avalible(self, alert_message: str, valid_choices: list[str]) -> Optional[str]:
+        while True:
+            choice = input(alert_message)
+            if choice in valid_choices:
+                return choice
+            else:
+                print(f"Invalid choice, available choices: {','.join(valid_choices)}. Please try again.")
+
+    def finalize_options(self):
+        if self.installed_browsers.__len__() == 0:
+            print("warning: to use preparser, you need at least one of the preparser browsers installed .")
+            install_browser = self.init_install_browser()
+            self.installed_browsers.append(install_browser)
+
+
+    def run(self):
+        print("checking weather there were preparser browsers' core installed .....")
+        self.precheck_installed_browsers()
+        recheck_browsers_number  = len(self.need_recheck_browsers)
+        total_browsers_number = len(self.browser_list)
+        if recheck_browsers_number > 0:
+            if recheck_browsers_number < total_browsers_number:
+                print(f'there were browsers of preparser not installed {",".join(self.need_recheck_browsers)}.')
+                print(f"warning: added more or not won't effect your next process, as you have installed {",".join(self.installed_browsers)} ")
+                choice = self.check_choice_avalible('do you still want to add them ? (yes/no) : ',['yes','no'])
+                if choice == 'yes':
+                    for browser in self.need_recheck_browsers:
+                        to_install_choice = self.check_choice_avalible(f'do you want to install {browser} of preparser ? (yes/no) : ',['yes','no'])
+                        if to_install_choice == 'yes':
+                            self.operate_browser("install",browser)
+                            self.installed_browsers.append(install_browser)
+            else: # no browser installed 
+                install_browser = self.init_install_browser()
+                self.installed_browsers.append(install_browser)
+
+        
+class PreInstaller(install):
+    def run(self):
+        print('Prechecking the environment status, before install preparser!!!')
+        if 'bdist_wheel' in sys.argv or 'build' in sys.argv:
+            # Avoid running the browser installer during the build phase
+            print("Skipping browser core installation during build.")
+        else:
+            # precheck the environment status before install
+            print('Prechecking the environment status, before install preparser!!!')
+            # Execute playwright install command
+            BrowserCoreInstaller(self.distribution).run()
+            print("All prechecks finished, begin installing preparser...")
+        # excute the default running
+        self.run(self)
+                
+        
+

MANIFEST.in

@@ -1,6 +1,6 @@
 include README.md
 include LICENSE
-
+include InstallHelper.py
 # include setup.py   # no need include current package, which will auto include the setup.py file when uploaded into PyPI
 
 # get ride of the Pipfile files, as it just been use for the virtual environment building

README.md

@@ -4,7 +4,7 @@ this is a sight Parser to help you pre_parser the datas from `specified website
 
 # Attention
 
-as this slight pre_parser  was just based the module of the `requests` and `beautifulsoup4`, which mainly was used to parse the data from the `api` and page randered as `static html`, so it can't directly parsed datas that need waiting the whole website pages was loaded, but for this function , maybe I will added later in future.
+as this slight pre_parser  for the old version 1.0.0, which only can help preparser the `static html` or `api` inform, but now from the 2.0.0 , I have added an new `html_dynamic` mode, which will help get all inform even generated by the `JS` code.
 
 ```bash
 
@@ -37,7 +37,7 @@ here below are some of the parameters you can use for initai the Object `PrePars
 | ---------------------  | -----------------   |--------------------------------------------------------   |
 | url_list               | list                | The list of URLs to parse from. Default is an empty list. |
 | request_call_back_func | Callable or None    | A callback function according to the parser_mode to handle the `BeautifulSoup` object or request `json` Object. and if you want to show your business process failed, you can return `None`, otherwise please return a `not None` Object.        |
-|  parser_mode           | `'html'` or `'api'` | The pre-parsing datas mode,default is `'html'`.<br/>  `html`: use the bs4 to parse the datas, and return an `BeautifulSoup` Object. <br/> `api` :  use requests only and return an `json` object. <br/>  **and all of them you can get it when you set the `request_call_back_func`, otherwise get it via the object of `PreParer(....).cached_request_datas`    |
+|  parser_mode           | `'html'`, `'api'` or `'html_dynamic'` | The pre-parsing datas mode,default is `'html'`.<br/>  `html`: parse the content from static html, and return an `BeautifulSoup` Object. <br/> `api`: parse the datas from an api, and return the `json` Object. <br/> `html_dynamic`: parse  from  the whole webpage html content and return an `BeautifulSoup` Object, even the content that generated by the dynamic js code. <br/>  **and all of Object you can get when you defined the `request_call_back_func`, otherwise get it via the object of `PreParer(....).cached_request_datas`    |
 | cached_data | bool | weather cache the parsed datas, defalt is False. |
 | start_threading | bool | Whether to use threading pool for parsing the data. Default is `False`.|
 | threading_mode | `'map'` or `'single'` | to run the task mode, default is `single`. <br/>  `map`: use the `map` func of the theading pool to distribute tasks. <br/> `single`: use the `submit` func to distribute the task one by one into the theading pool. |
@@ -94,7 +94,7 @@ if __name__ == "__main__":
     parser.start_parse()
 
     # when all task finished, you can get the all task result result like below:
-    all_results = parser.cached_request_datas
+    all_result = parser.cached_request_datas
 
     # if you want to terminal, just execute the function here below
     # parser.stop_parse()
@@ -111,3 +111,19 @@ if __name__ == "__main__":
 
 Get help ➡️ [Github issue](https://github.com/BertramYe/preparser/issues)
 
+
+# Update logs
+
+* `version 2.0.5 `: remove the dynamic mode browser core install from setup into package call.
+
+* `version 2.0.4 `: test the installing process command.
+
+* `version 2.0.3 `: optimise the `error` alert for `html_dynamic`.
+
+* `version 2.0.2 `: correct the README Doc of `parser_mode`.
+
+* `version 2.0.1 `: update the README Doc.
+
+* `version 2.0.0 `: add the new `parser_mode` of the `html_dynamic`, which help `preparser` all of the content from `html` , event it generated by the `JS` code.
+
+* `version 1.0.0 `: basical version, only `perparser` the static `html` and `api` content.

preparser/DynamicHelper.py

@@ -0,0 +1,83 @@
+
+import sys
+from os import path
+from subprocess import check_call
+from typing import Literal,Optional
+from playwright.sync_api import sync_playwright 
+
+class Dynamicer():
+    """
+        install the Browser Core
+    """
+    def __init__(self) -> None:
+        self.browser_list = ['chromium','firefox','webkit']
+        self._async_index = -1
+        
+    def _check_dynamic_async_env(self) -> int:
+        installed_browser_index = -1
+        try:
+            with sync_playwright() as p:
+                browser_Bundle_List = [
+                        p.chromium,
+                        p.firefox,
+                        p.webkit
+                ]
+                for i,browser_budle in enumerate(browser_Bundle_List):
+                    if path.exists(browser_budle.executable_path):
+                        installed_browser_index = i
+                        break
+            if installed_browser_index == -1:
+                installed_browser_index = self.init_install_browser()
+            # else:
+                # because so far is in the use checking, so no need add the re-install logical
+                # fro the precheck and install will added into the setup logical in the future
+        except Exception as error:
+            print(f"error: when check the preparser browser bundle, error:{error} !!!")
+            print(f'please try again, if failed again, please reinstall preparser !!!')
+        finally:
+            self._async_index = installed_browser_index
+            return installed_browser_index
+    
+    def _get_dynamic_html(self,url:str) -> str | None:
+        try:
+            if 0 <= self._async_index < 3:
+                with sync_playwright() as p:
+                    if self._async_index == 0:
+                        browser = p.chromium.launch(headless=True)
+                    elif self._async_index == 1:
+                        browser = p.firefox.launch(headless=True)
+                    else:
+                        browser = p.webkit.launch(headless=True)
+                    page = browser.new_page()
+                    page.goto(url)
+                    html = page.content()
+                    browser.close()
+                    return html
+            else:
+                return None
+        except Exception as err:
+            print(f'error when doing dynamic html parse , error: {err} !')
+            return None
+
+    def init_install_browser(self):
+        # if not , just let the user to choose
+        print("please choose a preparser browser  to install: ")
+        print("[1] chromium, [2] firefox, [3] webkit.")
+        choice = self.check_choice_avalible(f'please input a number to choose a browser (1/2/3):',['1','2','3'])
+        browser = self.browser_list[int(choice)-1]
+        self.operate_browser("install",browser)
+        return int(choice)-1         
+
+    def operate_browser(self,command_type:Literal["install","uninstall"], browser_name:str):
+        # install specified browser
+        print(f"{command_type}ing preparser browser {browser_name} ...")
+        check_call([sys.executable, "-m", "playwright", command_type, browser_name])    
+
+    def check_choice_avalible(self, alert_message: str, valid_choices: list[str]) -> Optional[str]:
+        while True:
+            choice = input(alert_message)
+            if choice in valid_choices:
+                return choice
+            else:
+                print(f"Invalid choice, available choices: {','.join(valid_choices)}. Please try again.")
+