url_to_html.py

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options

from webdriver_manager.chrome import ChromeDriverManager

from bs4 import BeautifulSoup
import re


import time



def sign_in_to_linkedin(driver,email, password):
    # Navigate to the LinkedIn login page
    driver.get('https://www.linkedin.com/login')
    # Enter your email address and password
    email_input = driver.find_element(By.ID, 'username')
    password_input = driver.find_element(By.ID, 'password')
    email_input.send_keys(email)
    password_input.send_keys(password)
    password_input.send_keys(Keys.ENTER)

def get_html_from_linkedin_url(driver,url:None)->str:
    try:
        driver.get(url)
        time.sleep(5)  # Allow time for the page to load and render JavaScript
        html_content = driver.page_source
    except Exception as e:
        print(f"Error with Selenium: {e}")
    finally:
        driver.quit() # Close the browser

    return html_content

def extract_text_from_html(html: str) -> str:
    """Try to extract the main job description from LinkedIn job HTML. This is heuristic."""
    soup = BeautifulSoup(html, "html.parser")
    # LinkedIn job description often sits in elements with class names containing 'description' or 'job-description'
    candidates = soup.find_all(lambda tag: tag.name in ("div", "section") and tag.get("class"))
    best_text = []
    for c in candidates:
        classes = " ".join(c.get("class") or [])
        if "description" in classes or "job" in classes or "job-description" in classes or "description__text" in classes:
            text = c.get_text(separator="\n", strip=True)
            if text and len(text) > 200:
                best_text.append(text)
    # fallback: whole page text
    if not best_text:
        return soup.get_text(separator="\n", strip=True)
    # choose the largest block
    best_text.sort(key=len, reverse=True)
    return best_text[0]

def job_text_for_linkedin_url(url: str) -> str:
    """Fetch HTML from a LinkedIn job URL."""

    # Set up Chrome options for headless browsing (optional)
    chrome_options = Options()
    chrome_options.add_argument("--headless")  # Run Chrome in headless mode (no UI)
    chrome_options.add_argument("--disable-gpu") # Recommended for headless mode

    # Set up ChromeDriver
    driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=chrome_options)

    # Sign in to LinkedIn
    sign_in_to_linkedin(driver, 'sample_email', 'sample_password')

    # Get HTML from LinkedIn job URL
    html_content = get_html_from_linkedin_url(driver,url)
    
    return extract_text_from_html(html_content)