add time.sleep(10) to avoid getting 429 status and do refactor

Author: lenarsaitov

cianparser/base.py cianparser/base_list.pycianparser/base.py renamed to cianparser/base_list.py

@@ -7,10 +7,10 @@
 class BaseListPageParser:
     def __init__(self,
                  session,
-                 deal_type: str, rent_period_type, location_name: str,
+                 accommodation_type: str, deal_type: str, rent_period_type, location_name: str,
                  with_saving_csv=False, with_extra_data=False,
                  object_type=None, additional_settings=None):
-        self.accommodation_type = "suburban"
+        self.accommodation_type = accommodation_type
         self.session = session
         self.deal_type = deal_type
         self.rent_period_type = rent_period_type

cianparser/cianparser.py

@@ -1,11 +1,12 @@
 import cloudscraper
+import time
 
 from cianparser.constants import CITIES, METRO_STATIONS, DEAL_TYPES, OBJECT_SUBURBAN_TYPES
 from cianparser.url_builder import URLBuilder
-from cianparser.flat_list import FlatListPageParser
-from cianparser.suburban_list import SuburbanListPageParser
-from cianparser.newobject_list import NewObjectListParser
 from cianparser.proxy_pool import ProxyPool
+from cianparser.flat.list import FlatListPageParser
+from cianparser.suburban.list import SuburbanListPageParser
+from cianparser.newobject.list import NewObjectListParser
 
 
 def list_locations():
@@ -42,14 +43,16 @@ def __set_proxy__(self, url_list):
         if available_proxy is not None:
             self.__session__.proxies = {"https": available_proxy}
 
-    def __load_list_page__(self, url_list_format, page_number):
+    def __load_list_page__(self, url_list_format, page_number, attempt_number_exception):
         url_list = url_list_format.format(page_number)
         self.__set_proxy__(url_list)
 
-        if page_number == self.__parser__.start_page:
+        if page_number == self.__parser__.start_page and attempt_number_exception == 0:
             print(f"The page from which the collection of information begins: \n {url_list}")
 
         res = self.__session__.get(url=url_list)
+        if res.status_code == 429:
+            time.sleep(10)
         res.raise_for_status()
 
         return res.text
@@ -70,7 +73,7 @@ def __run__(self, url_list_format: str):
             while attempt_number_exception < 3 and not page_parsed:
                 try:
                     (page_parsed, attempt_number, end_all_parsing) = self.__parser__.parse_list_offers_page(
-                        html=self.__load_list_page__(url_list_format=url_list_format, page_number=page_number),
+                        html=self.__load_list_page__(url_list_format=url_list_format, page_number=page_number, attempt_number_exception=attempt_number_exception),
                         page_number=page_number,
                         count_of_pages=self.__parser__.end_page + 1 - self.__parser__.start_page,
                         attempt_number=attempt_number_exception)
@@ -105,6 +108,7 @@ def get_flats(self, deal_type: str, rooms, with_saving_csv=False, with_extra_dat
         deal_type, rent_period_type = __define_deal_type__(deal_type)
         self.__parser__ = FlatListPageParser(
             session=self.__session__,
+            accommodation_type="flat",
             deal_type=deal_type,
             rent_period_type=rent_period_type,
             location_name=self.__location_name__,
@@ -137,6 +141,7 @@ def get_suburban(self, suburban_type: str, deal_type: str, with_saving_csv=False
         deal_type, rent_period_type = __define_deal_type__(deal_type)
         self.__parser__ = SuburbanListPageParser(
             session=self.__session__,
+            accommodation_type="suburban",
             deal_type=deal_type,
             rent_period_type=rent_period_type,
             location_name=self.__location_name__,

cianparser/utils/__init__.py cianparser/definers/__init__.pycianparser/utils/__init__.py renamed to cianparser/definers/__init__.py

@@ -6,8 +6,8 @@
 
 from cianparser.constants import FILE_NAME_FLAT_FORMAT
 from cianparser.helpers import union_dicts, define_author, define_location_data, define_specification_data, define_deal_url_id, define_price_data
-from cianparser.flat import FlatPageParser
-from cianparser.base import BaseListPageParser
+from cianparser.flat.page import FlatPageParser
+from cianparser.base_list import BaseListPageParser
 
 
 class FlatListPageParser(BaseListPageParser):

cianparser/utils/definer_cities_id.py cianparser/definers/definer_cities_id.pycianparser/utils/definer_cities_id.py renamed to cianparser/definers/definer_cities_id.py

@@ -1,5 +1,6 @@
 import bs4
 import re
+import time
 
 
 class FlatPageParser:
@@ -9,6 +10,8 @@ def __init__(self, session, url):
 
     def __load_page__(self):
         res = self.session.get(self.url)
+        if res.status_code == 429:
+            time.sleep(10)
         res.raise_for_status()
         self.offer_page_html = res.text
         self.offer_page_soup = bs4.BeautifulSoup(self.offer_page_html, 'html.parser')

cianparser/utils/definer_metro_id.py cianparser/definers/definer_metro_id.pycianparser/utils/definer_metro_id.py renamed to cianparser/definers/definer_metro_id.py

@@ -9,7 +9,7 @@
 
 from cianparser.constants import FILE_NAME_NEWOBJECT_FORMAT
 from cianparser.helpers import union_dicts
-from cianparser.newobject import NewObjectPageParser
+from cianparser.newobject.page import NewObjectPageParser
 
 
 class NewObjectListParser:

cianparser/flat_list.py cianparser/flat/list.pycianparser/flat_list.py renamed to cianparser/flat/list.py

@@ -1,5 +1,6 @@
 import bs4
 import re
+import time
 
 
 class NewObjectPageParser:
@@ -9,6 +10,8 @@ def __init__(self, session, url):
 
     def __load_page__(self):
         res = self.session.get(self.url)
+        if res.status_code == 429:
+            time.sleep(10)
         res.raise_for_status()
         self.offer_page_html = res.text
         self.offer_page_soup = bs4.BeautifulSoup(self.offer_page_html, 'html.parser')

cianparser/flat.py cianparser/flat/page.pycianparser/flat.py renamed to cianparser/flat/page.py

@@ -6,8 +6,8 @@
 
 from cianparser.constants import FILE_NAME_SUBURBAN_FORMAT
 from cianparser.helpers import union_dicts, define_author, parse_location_data, define_price_data, define_deal_url_id
-from cianparser.suburban import SuburbanPageParser
-from cianparser.base import BaseListPageParser
+from cianparser.suburban.page import SuburbanPageParser
+from cianparser.base_list import BaseListPageParser
 
 
 class SuburbanListPageParser(BaseListPageParser):