added databricks

Author: rabbit0021

db/sqlite.py

@@ -314,6 +314,7 @@ def add_publisher(self, conn, publisher_name, publisher_type,):
             VALUES (?, ?)
         """, (publisher_name, publisher_type))
         logger.info(f"Publisher {publisher_name} added successfully")
+        return c.lastrowid
 
     def update_publisher(self, conn, publisher_id, last_scraped_at):
         logger.info(f"Updating publisher: {publisher_id}, last_scraped_at: {last_scraped_at}")
@@ -372,6 +373,16 @@ def get_posts(self, conn):
         rows = c.fetchall()
         return [dict(row) for row in rows]
 
+    def get_posts_by_publisher_id(self, conn, pub_id):
+        c = conn.cursor()
+        c.execute("""
+            SELECT *
+            FROM posts
+            WHERE publisher_id = ?
+        """, (pub_id,))
+        rows = c.fetchall()
+        return [dict(row) for row in rows]
+    
     def update_post_label(self, conn, post_id, label):
         logger.info(f"Updating post: {post_id}, with label: {label}")
         c = conn.cursor()

handlers/databricks.py

@@ -0,0 +1,7 @@
+from .base import BaseScraper
+
+BASE_URL = "https://databricks.com/feed"
+class DatabricksScraper(BaseScraper):
+    
+    def get_feed_url(self):
+        return BASE_URL

handlers/factory.py

@@ -1,4 +1,4 @@
-from handlers import aws, github, netflix, airbnb, dropbox, facebook, slack, spotify, cloudfare, nvidea, salesforce, google
+from handlers import aws, github, netflix, airbnb, dropbox, facebook, slack, spotify, cloudfare, nvidea, salesforce, google, databricks
 from logger_config import get_logger;
 
 logger = get_logger("HANDLERS")
@@ -29,6 +29,8 @@ def get_scraper(comapny):
             return salesforce.SalesforceScraper()
         elif comapny.lower() == 'google':
             return google.GoogleScraper()
+        elif comapny.lower() == "databricks":
+            return databricks.DatabricksScraper()
         else:
             logger.error(f"No handler found for {comapny}")
             return None

tests/e2e/test_scrape_pubs.py

@@ -5,20 +5,35 @@
 from scrape_pubs import scrape_pubs
 
 @pytest.mark.pubs
-def test_scrape_pubs_techteams(db):
+def test_scrape_pubs_techteams1(db):
     conn = db.get_connection()
 
-    db.add_publisher(conn, "facebook", "techteam")
-    db.add_subscription(conn, "newemail5@gmail.com", enums.PublisherCategory.SOFTWARE_ENGINEERING.value, 1)
+    pubid = db.add_publisher(conn, "facebook", "techteam")
+    db.add_subscription(conn, "newemail5@gmail.com", enums.PublisherCategory.SOFTWARE_ENGINEERING.value, pubid)
 
     scrape_pubs(db, conn)
 
-    posts = db.get_posts(conn)
+    posts = db.get_posts_by_publisher_id(conn, pubid)
 
     assert len(posts) > 10
 
     conn.close()
 
+@pytest.mark.pubs
+def test_scrape_pubs_techteams2(db):
+    conn = db.get_connection()
+    
+    pubid = db.add_publisher(conn, "databricks", "techteam")
+    db.add_subscription(conn, "newemail6@gmail.com", enums.PublisherCategory.SOFTWARE_ENGINEERING.value, pubid)
+    
+    scrape_pubs(db, conn)
+    
+    posts = db.get_posts_by_publisher_id(conn, pubid)
+    
+    assert len(posts) > 9
+    
+    conn.close()
+