feat: newly opened companies; persist company list

Author: proffapt

docker-compose.yml

@@ -13,6 +13,7 @@ services:
             - $MFTP_CONFIG/mail_send_token.json:/app/mail_send_token.json
             - $MFTP_CONFIG/mail_send_creds.json:/app/mail_send_creds.json
             - $MFTP_CONFIG/.session:/app/.session
+            - $MFTP_CONFIG/companies.json:/app/companies.json
         depends_on:
             - db
         networks:

mftp/company.py

@@ -1,3 +1,5 @@
+import os
+import json
 import logging
 from env import ROLL_NUMBER
 from datetime import datetime
@@ -6,6 +8,9 @@
 from endpoints import TPSTUDENT_URL, COMPANIES_URL
 
 
+COMPANIES_FILE = f"{os.path.dirname(__file__)}/companies.json"
+
+
 def filter(companies, filter):
     print('[FILTERING COMPANY UPDATES]', flush=True)
 
@@ -44,7 +49,7 @@ def fetch(session, headers, ssoToken):
     xml_encoded = xml_string.encode("utf-8")
     root = ET.fromstring(xml_encoded)
 
-    companies = []
+    fetched_companies = []
     for row in root.findall("row"):
         jd_args = row.find("cell[4]").text.split("'")[5].split('"')
         jnf_id, com_id, year = jd_args[1], jd_args[3], jd_args[5]
@@ -75,9 +80,52 @@ def fetch(session, headers, ssoToken):
             "Interview_Date": row.find("cell[12]").text.strip() if row.find("cell[12]").text.strip() else None,
         }
 
-        companies.append(company_info)
+        fetched_companies.append(company_info)
+    
+    stored_companies = get_list()
+    new_companies, modified_companies = get_new_and_modified_companies(fetched_companies, stored_companies)
+
+    store_list(fetched_companies)
+
+    return fetched_companies, new_companies, modified_companies
+
+
+def get_new_and_modified_companies(fetched, stored, unique_key="Job_Description"):
+    # Create dictionaries for quick lookup by the unique key
+    stored_dict = {entry[unique_key]: entry for entry in stored}
+    fetched_dict = {entry[unique_key]: entry for entry in fetched}
+
+    new_entries = []
+    updated_entries = []
+
+    for key, fetched_entry in fetched_dict.items():
+        if key not in stored_dict:
+            # New entry
+            new_entries.append(fetched_entry)
+        else:
+            # Compare the values of the fetched entry with the stored entry
+            stored_entry = stored_dict[key]
+            if any(fetched_entry[k] != stored_entry.get(k) for k in fetched_entry):
+                updated_entries.append(fetched_entry)
+
+    return new_entries, updated_entries
+
+
+def store_list(companies):
+    with open(COMPANIES_FILE, "w") as json_file:
+        json.dump(companies, json_file, indent=2)
+
 
-    return companies
+def get_list():
+    try:
+        with open(COMPANIES_FILE, "r") as json_file:
+            return json.load(json_file)
+    except json.JSONDecodeError as _:
+        store_list([])
+        return []
+    except FileNotFoundError:
+        store_list([])
+        return []
 
 
 # Downloads pdf content in bytes format

mftp/docker-compose.yml

@@ -13,6 +13,7 @@ services:
             - $MFTP_CONFIG/mail_send_token.json:/app/mail_send_token.json
             - $MFTP_CONFIG/mail_send_creds.json:/app/mail_send_creds.json
             - $MFTP_CONFIG/.session:/app/.session
+            - $MFTP_CONFIG/companies.json:/app/companies.json
         depends_on:
             - db
         networks:

mftp/mail.py

@@ -56,11 +56,11 @@ def send_companies(mail, gmail_api, smtp):
                 logging.error(f" Failed to Send Mail : {mail['Subject']} ~ {str(e)}")
 
 
-def format_companies(ssoToken, companies):
+def format_companies(ssoToken, companies, subject):
     print('[FORMATTING COMPANY UPDATES]', flush=True)
 
     message = MIMEMultipart()
-    message["Subject"] = "APPLY NOW! | Companies Open = Y & Applied = N "
+    message["Subject"] = subject
     message["From"] = f'MFTP < {FROM_EMAIL} >'
     message["Bcc"] = ", ".join(HOSTER_EMAIL)
 

mftp/mftp.py

@@ -6,6 +6,7 @@
 import notice
 import company
 
+import logging
 import requests
 import argparse
 from datetime import datetime
@@ -32,11 +33,30 @@
   _, ssoToken = erp.login(headers, session, ERPCREDS=env, OTP_CHECK_INTERVAL=2, LOGGING=True, SESSION_STORAGE_FILE='.session')
 
   if args.gmail_api or args.smtp:
-    if now.minute == 0:
-      companies = company.fetch(session, headers, ssoToken)
-      open_not_applied_companies = company.filter(companies, "OPEN_N")
-      companies_update_mail = mail.format_companies(session.cookies.get('ssoToken'), open_not_applied_companies)
-      mail.send_companies(companies_update_mail, args.gmail_api, args.smtp)
+    _, new, modified = company.fetch(session, headers, ssoToken)
+
+    if new:
+      print('[NEW COMPANIES]', flush=True)
+      for com in new:
+        logging.info(f' {com["Name"]} | {com["Role"]} | {com["CTC"]} | {com["End_Date"]} | {com["Interview_Date"]}')
+    if modified:
+      print('[MODIFIED COMPANIES]', flush=True)
+      for com in modified:
+        logging.info(f' {com["Name"]} | {com["Role"]} | {com["CTC"]} | {com["End_Date"]} | {com["Interview_Date"]}')
+
+    filtered = []
+    if new + modified:
+      filtered = company.filter(new + modified, "OPEN_N")
+      if filtered:
+        for com in filtered:
+          logging.info(f' {com["Name"]} | {com["Role"]} | {com["CTC"]} | {com["End_Date"]} | {com["Interview_Date"]}')
+
+          latest_ssoToken = session.cookies.get('ssoToken')
+          mail_subject = "APPLY NOW! New companies opened"
+          companies_mail = mail.format_companies(latest_ssoToken, filtered, mail_subject)
+          mail.send_companies(companies_mail, args.gmail_api, args.smtp)
+    else:
+      print("[NO NEW COMPANIES]")
 
   notice_db = db.NoticeDB(config={
     'uri': env.MONGO_URI,