Enhance scraper.py with additional debug output

- Add print statement to output the response text from the main listing page
- Improve visibility into the data being fetched during the scraping process

Author: Adriano Sanges

real-estate-etl/scraper.py

@@ -34,7 +34,6 @@ def parse_page(url: str) -> Dict[str, Optional[any]]:
     listings = soup.select('section.ld-layoutContentCenter')
     for listing in listings:
         # Extract title, city, neighborhood, road
-        print(listing)
         title = soup.find('meta', property='og:title')
         city, neighbourhood, road = None, None, None
         if title:
@@ -102,6 +101,7 @@ def parse_listing(url: str) -> List[Dict[str, Optional[any]]]:
     }
     logging.debug("Fetching main listing page: %s", url)
     response = requests.get(url, headers=headers)
+    print(response.text)
     soup = BeautifulSoup(response.text, 'html.parser')
     data_list = []
     links = soup.select('a.in-listingCardTitle')