Add garage information parsing to property scraping and database schema

Author: Adriano Sanges

real-estate-etl/database.py

@@ -13,7 +13,8 @@ def clean_properties(con) -> None:
     neighbourhood TEXT,
     road TEXT,
     square_meters INTEGER,
-    floor TEXT
+    floor TEXT,
+    garage_info TEXT
 );
 CREATE OR REPLACE TABLE main.new_properties (
     url TEXT PRIMARY KEY,
@@ -24,13 +25,14 @@ def clean_properties(con) -> None:
     neighbourhood TEXT,
     road TEXT,
     square_meters INTEGER,
-    floor TEXT
+    floor TEXT,
+    garage_info TEXT
 );
 """
     con.sql(create_table_query)
     insert_query = """
-INSERT INTO main.cleaned_properties (url, title, price, city, neighbourhood, road, square_meters, floor)
-SELECT url, title, price, city, neighbourhood, road, square_meters, floor
+INSERT INTO main.cleaned_properties (url, title, price, city, neighbourhood, road, square_meters, floor, garage_info)
+SELECT url, title, price, city, neighbourhood, road, square_meters, floor, garage_info
 FROM main.properties
 WHERE NOT EXISTS (  
     SELECT 1 
@@ -39,8 +41,8 @@ def clean_properties(con) -> None:
 );
 """
     insert_query_only_new = """
-INSERT INTO main.new_properties (url, title, price, city, neighbourhood, road, square_meters, floor)
-SELECT url, title, price, city, neighbourhood, road, square_meters, floor
+INSERT INTO main.new_properties (url, title, price, city, neighbourhood, road, square_meters, floor, garage_info)
+SELECT url, title, price, city, neighbourhood, road, square_meters, floor, garage_info
 FROM main.properties
 WHERE NOT EXISTS (  
     SELECT 1 

real-estate-etl/scan_properties.py

@@ -1,7 +1,4 @@
-import requests
 from bs4 import BeautifulSoup
-import re
-import logging
 import polars
 import duckdb
 from scraper import parse_listing

real-estate-etl/scraper.py

@@ -29,6 +29,16 @@ def parse_page(url):
 
         floor_match = re.search(r'Piano\s(\d+)', soup.text)
         floor = int(floor_match.group(1)) if floor_match else None
+
+                # Find the feature item related to parking/garage
+        garage_feature = listing.find('dt', class_='re-featuresItem__title', string="Box, posti auto")
+
+        if garage_feature:
+            # Get the associated description (dd)
+            garage_description = garage_feature.find_next('dd', class_='re-featuresItem__description')
+            garage_info = garage_description.get_text(strip=True) if garage_description else None
+        else:
+            garage_info = None
 
         data = {
             "url": url,
@@ -40,6 +50,7 @@ def parse_page(url):
             "road": road.text.strip() if road else None,
             "square_meters": square_meters,
             "floor": floor,
+            "garage_info": garage_info,
         }
 
     return data

real-estate-etl/telegram_api.py

@@ -1,11 +1,15 @@
 import telegram
 import os
 
+from dotenv import load_dotenv
 
-TELEGRAM_BOT_API_KEY = os.getenv('telegram_bot_api_key')
+load_dotenv()
+
+telegram_bot_api_key = os.getenv('telegram_bot_api_key')
 chat_id = os.getenv('chat_id')
 chat_tag = os.getenv('chat_tag')
-bot = telegram.Bot(TELEGRAM_BOT_API_KEY)
+
+bot = telegram.Bot(telegram_bot_api_key)
 
 
 # Function to format the message