added training classifier

Author: rabbit0021

app.py

@@ -8,10 +8,11 @@
 from datetime import timezone
 from db import get_database
 import time
+from functools import wraps
 
 app = Flask(__name__, static_folder="static", template_folder="templates")
-app.config['SECRET_KEY'] = os.getenv('SECRET_KEY', 'default-secret')
 app.db = get_database()
+SECRET_KEY = os.getenv("POSTS_SECRET_KEY", "******")
 
 # Logging    
 app.logger = get_logger("app")
@@ -24,6 +25,15 @@
 # test log
 register_middlewares(app)
 
+def require_secret_key(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        key = request.headers.get("X-SECRET-KEY")
+        if key != SECRET_KEY:
+            return jsonify({"status": "error", "message": "Unauthorized"}), 401
+        return f(*args, **kwargs)
+    return decorated
+
 @app.route("/")
 def index():
     return render_template("index.html", time=time.time)
@@ -48,6 +58,10 @@ def subscribe():
     techteams = data.get('techteams')
     individuals = data.get('individuals')
     communities = data.get('communities')
+    frequency = data.get('frquency') 
+    
+    if not frequency:
+        frequency = 3
 
     if not email or not topic or (not techteams and not individuals and not communities):
         return jsonify({"status": "error", "message": "Missing email or topic or publisher"
@@ -68,7 +82,7 @@ def subscribe():
 
                 existing_subscriptions = app.db.get_subscriptions_by_email(conn, email)
                 if not any(sub["publisher"]["id"] == publisher["id"] and sub["topic"] == topic for sub in existing_subscriptions):
-                    app.db.add_subscription(conn, email, topic, publisher['id'])
+                    app.db.add_subscription(conn, email, topic, publisher['id'], frequency=frequency)
 
         conn.commit()
         return jsonify({
@@ -163,6 +177,47 @@ def robots_txt():
 def sitemap_xml():
     return send_from_directory(app.static_folder, "sitemap.xml")
 
+@app.route("/postview.html")
+def postview():
+    return send_from_directory(app.template_folder, "posts.html")
+        
+@app.route("/posts", methods=["GET"])
+@require_secret_key
+def get_posts():
+    conn = app.db.get_connection()
+    try:
+        posts = app.db.get_posts(conn)
+        result = []
+        for post in posts:
+            result.append({
+                "id": post["id"],
+                "url": post["url"],
+                "title": post["title"],
+                "topic": post["topic"],
+                "labelled": post['labelled']
+            })
+        return jsonify(result)
+    finally:
+        conn.close()
+        
+@app.route("/posts/<int:post_id>", methods=["PATCH"])
+def update_post(post_id):
+    key = request.headers.get("X-SECRET-KEY")
+    if key != SECRET_KEY:
+        return jsonify({"status": "error", "message": "Unauthorized"}), 401
+
+    data = request.get_json()
+    topic = data.get("topic")
+    if not topic:
+        return jsonify({"status": "error", "message": "No topic provided"}), 400
+
+    conn = app.db.get_connection()
+    try:
+        app.db.update_post_label(conn, post_id, topic)
+        return jsonify({"status": "success", "message": f"Post {post_id} updated"})
+    finally:
+        conn.close()
+        
 if __name__ == "__main__":
     if os.getenv("FLASK_ENV") == "Production":
         app.run()

classifier.py

@@ -1,11 +1,19 @@
+# classifier_model.py
+import os
 from sentence_transformers import SentenceTransformer, util
 from db import enums
-import torch
+import pickle
+from logger_config import get_logger
 
-# 1. Load a stronger model
-model = SentenceTransformer('all-mpnet-base-v2')
+env = os.getenv('FLASK_ENV', 'development')
+MODEL_PATH = os.getenv("MODEL_PATH") if env == 'production' else 'data/dev/trained_classifier.pkl'
+CONFIDENCE_THRESHOLD = 0.7
 
-# 2. Concise category descriptions
+logger = get_logger("classifier")
+# Load embedding model
+embedding_model = SentenceTransformer('all-mpnet-base-v2')
+
+# Category descriptions
 categories = {
     enums.PublisherCategory.SOFTWARE_ENGINEERING.value: (
         "frontend, backend, APIs, microservices, databases, relational databases, cloud databases, DevOps, system design, CI/CD, containers, scalability, performance, distributed systems, mobile, UI/UX"
@@ -27,17 +35,17 @@
     )
 }
 
-# 3. Encode category descriptions
+# Precompute embeddings for baseline
 category_embeddings = {
-    cat: model.encode(desc, convert_to_tensor=True)
+    cat: embedding_model.encode(desc, convert_to_tensor=True)
     for cat, desc in categories.items()
 }
 
-# Optional: simple keyword mapping to override embeddings
+# Optional keyword mapping
 keywords_map = {
     enums.PublisherCategory.SOFTWARE_ENGINEERING.value: [
-        "react", "angular", "vue", "node.js", "django", "java", "go", 
-        "microservices", "api", "devops", "kubernetes", 
+        "react", "angular", "vue", "node.js", "django", "java", "go",
+        "microservices", "api", "devops", "kubernetes",
         "aurora", "rds", "cloud database", "postgresql", "mysql", "mongodb", "redis", "database"
     ],
     enums.PublisherCategory.SOFTWARE_TESTING.value: [
@@ -54,41 +62,56 @@
     ]
 }
 
-def classify_post(post_title, tags="", content=""):
-    """
-    Classify a post into a category using title + tags + first 100 chars of content.
-    Uses embeddings similarity with optional keyword boost.
-    """
+# ===== Load trained classifier if exists =====
+trained_clf = None
+label_encoder = None
+os.makedirs(os.path.dirname(MODEL_PATH), exist_ok=True)
 
-    # 1. Prepare text
-    content_snippet = content[:100] if content else ""
-    combined_text = f"Title: {post_title}. Tags: {tags}. Content: {content_snippet}".lower()
+if os.path.exists(MODEL_PATH):
+    with open(MODEL_PATH, "rb") as f:
+        trained_clf, label_encoder = pickle.load(f)
+    logger.info(f"[Classifier] Loaded trained classifier from {MODEL_PATH}")
 
-    # 2. Encode input
-    text_embedding = model.encode(combined_text, convert_to_tensor=True)
+# ===== Baseline classifier =====
+def classify_with_embeddings(title, tags="", content=""):
+    content_snippet = content[:100] if content else ""
+    combined_text = f"Title: {title}. Tags: {tags}. Content: {content_snippet}".lower()
+    text_embedding = embedding_model.encode(combined_text, convert_to_tensor=True)
 
-    # 3. Compute similarity
-    scores = {
-        cat: util.cos_sim(text_embedding, emb).item()
-        for cat, emb in category_embeddings.items()
-    }
+    scores = {cat: util.cos_sim(text_embedding, emb).item()
+              for cat, emb in category_embeddings.items()}
 
-    # 4. Keyword boost: add 0.1 if a keyword exists in title/tags/content
     combined_lower = combined_text.lower()
     for cat, kw_list in keywords_map.items():
         for kw in kw_list:
             if kw in combined_lower:
                 scores[cat] += 0.1
                 break
 
-    # 5. Assign category with highest similarity
     best_cat = max(scores, key=scores.get)
-
-    # 6. Adaptive fallback: check relative score
     sorted_scores = sorted(scores.values(), reverse=True)
     top_score = sorted_scores[0]
     second_score = sorted_scores[1] if len(sorted_scores) > 1 else 0.0
     if top_score < 0.25 or (top_score - second_score) < 0.05:
         return enums.PublisherCategory.GENERAL.value
-
     return best_cat
+
+# ===== Unified classifier =====
+def classify_post(title, tags="", content=""):
+    global trained_clf, label_encoder
+
+    if trained_clf and label_encoder:
+        logger.info("Attempt to use trained classifier")
+
+        text_embedding = embedding_model.encode(f"Title: {title}. Tags: {tags}. Content: {content[:100]}")
+        pred_proba = trained_clf.predict_proba([text_embedding])[0]
+        max_prob = pred_proba.max()
+        if max_prob >= CONFIDENCE_THRESHOLD:
+            logger.info("Good confidence score with trained classifier")
+            pred_label = trained_clf.predict([text_embedding])[0]
+            return label_encoder.inverse_transform([pred_label])[0]
+        else:
+            logger.info(f"Fallback to normal without trained mode due to low confidence: {max_prob}")
+
+    # fallback
+    return classify_with_embeddings(title, tags, content)

db/sqlite.py

@@ -230,15 +230,32 @@ def get_notifications_by_email(self, conn, email):
         """, (email,))
         rows = c.fetchall()
         return [dict(row) for row in rows]
+    
+    def get_active_notifications_by_email_and_url(self, conn, email, url):
+        c = conn.cursor()
+        c.execute("""
+            SELECT *
+            FROM notifications
+            WHERE email = ? and post_url = ? and deleted=0
+        """, (email,url))
+        row = c.fetchone()
+        return dict(row) if row else None
 
     def add_notification(self, conn, email, heading, style_version, post_url, post_title, maturity_date):
         logger.info(f"Adding notification: {email}, type: {post_title}")
-        c = conn.cursor()
-        c.execute("""
-            INSERT INTO notifications (email, heading, style_version, post_url, post_title, maturity_date)
-            VALUES (?, ?, ?, ?, ?, ?)
-        """, (email, heading, style_version, post_url, post_title, maturity_date))
-        logger.info("notification added successfully!")
+        
+        notf = self.get_active_notifications_by_email_and_url(conn, email, post_url)
+        
+        if not notf:
+            c = conn.cursor()
+            c.execute("""
+                INSERT INTO notifications (email, heading, style_version, post_url, post_title, maturity_date)
+                VALUES (?, ?, ?, ?, ?, ?)
+            """, (email, heading, style_version, post_url, post_title, maturity_date))
+            logger.info("notification added successfully!")
+        else:
+            logger.info("notification already existed!")
+
 
     def delete_notification(self, conn, email, post_url):
         logger.info(f"Deleting notification: {email}, url: {post_url}")

handlers/base.py

@@ -1,4 +1,3 @@
-from datetime import datetime
 import feedparser
 from datetime import timezone
 import ssl
@@ -8,7 +7,7 @@
 
 HEADERS = {'User-Agent': 'Mozilla/5.0'}
 
-logger = get_logger("handlers")
+logger = get_logger("base-handler")
 
 class BaseScraper:        
     def get_feed_url(self):
@@ -39,25 +38,28 @@ def search_blog_posts(self, category, last_scan_time):
 
             try:
                 # Parse published date using the correct format
-                published = parsedate_to_datetime(entry.published)
-            except ValueError as e:
-                logger.error(f"Date parse error: {entry.published} -> {e}")
-                continue
-            
-            if last_scan_time.tzinfo is None:
-                last_scan_time = last_scan_time.replace(tzinfo=timezone.utc)
-            if published <= last_scan_time:
-                logger.debug(f"Skipping {entry.title}: article published on {published} before last scan time: {last_scan_time}")
-                continue    
-            
-            # full_content = entry.content[0].value if entry.content else ""
-            # content = full_content[:100]  # truncate to first 100 chars
-
-            matching_posts.append({
-                "title": entry.title,
-                "url": entry.link,
-                "published": published.isoformat(),
-                "tags": categories
-            })    
+                published = None
+                if hasattr(entry, "published"):
+                    published = parsedate_to_datetime(entry.published)
+                elif hasattr(entry, "updated"):
+                    published = parsedate_to_datetime(entry.updated)
+                
+                if published is None:
+                   published = self.get_date_from_url(entry)
+                                                          
+                if last_scan_time.tzinfo is None:
+                    last_scan_time = last_scan_time.replace(tzinfo=timezone.utc)
+                if published <= last_scan_time:
+                    logger.debug(f"Skipping {entry.title}: article published on {published} before last scan time: {last_scan_time}")
+                    continue    
+                
+                matching_posts.append({
+                    "title": entry.title,
+                    "url": entry.link,
+                    "published": published.isoformat(),
+                    "tags": categories
+                })   
+            except Exception:
+                logger.exception(f"Date parse error: {entry}") 
 
         return matching_posts

handlers/google.py

@@ -1,8 +1,70 @@
 from .base import BaseScraper
+from datetime import datetime
+from datetime import timezone
+from logger_config import get_logger
+import requests
+from bs4 import BeautifulSoup
 
-BASE_URL = "https://blog.google/rss/"
+BASE_URL = "https://developers.googleblog.com/rss"
+
+logger = get_logger("google-handler")
 
 class GoogleScraper(BaseScraper):
-        
+    
     def get_feed_url(self):
-        return BASE_URL
+        return BASE_URL
+        
+    def parse_google_blog_date(self, date_str: str):
+        """
+        Parse Google Developers Blog dates like:
+        - 'AUG. 18, 2025'
+        - 'JULY 24, 2025'
+        Returns a timezone-aware datetime in UTC.
+        """
+        if not date_str:
+            return None    
+
+        # Clean string: remove dot, normalize case
+        clean_str = date_str.replace('.', '').title()  # 'Aug 18, 2025' or 'July 24, 2025'    
+
+        # Try full month name first (%B), then abbreviated (%b)
+        for fmt in ("%B %d, %Y", "%b %d, %Y"):
+            try:
+                dt = datetime.strptime(clean_str, fmt)
+                return dt.replace(tzinfo=timezone.utc)
+            except ValueError:
+                continue    
+
+        logger.warning(f"Unable to parse date from Google blog: '{date_str}'")
+        return None
+        
+    def get_date_from_url(self, entry):
+        """Fetch published date from Google Developers Blog post HTML."""
+        try:
+            url = entry.link
+            title = entry.title
+            resp = requests.get(url, timeout=5)
+            if resp.status_code != 200:
+                logger.warning(f"Non-200 response for {title}: {resp.status_code}")
+                return None    
+
+            soup = BeautifulSoup(resp.text, "html.parser")    
+
+            # Target the div with class "published-date glue-font-weight-medium"
+            div_date = soup.find("div", class_="published-date glue-font-weight-medium")
+            if div_date and div_date.text.strip():
+                published_text = div_date.text.strip()
+                logger.info(f"Published date for title {title}: {published_text}")
+                return self.parse_google_blog_date(published_text)    
+
+            # fallback to HTTP Last-Modified header
+            last_mod = resp.headers.get("Last-Modified")
+            if last_mod:
+                logger.info(f"Published date for title {title}: {last_mod}")
+                return self.parse_google_blog_date(last_mod)    
+
+        except Exception:
+            logger.exception(f"Failed to get published date from {url}")    
+
+        logger.info(f"No published date found for {url}")
+        return None