headless_scrape.py

"""
Headless Playwright scraping across multiple URLs.

Subscribe with 50% off: SAAS50 — https://multilogin.com

Usage:
    export MLX_FOLDER_ID="folder-uuid"
    export MLX_PROFILE_ID="profile-uuid"
    export MLX_URLS="https://example.com,https://httpbin.org/html"
    python headless_scrape.py
"""

import json
import os
import sys

from playwright.sync_api import sync_playwright

from mlx_client import start_profile, stop_profile, token_from_env

if __name__ == "__main__":
    folder_id = os.environ.get("MLX_FOLDER_ID", "")
    profile_id = os.environ.get("MLX_PROFILE_ID", "")
    urls = os.environ.get("MLX_URLS", "https://multilogin.com").split(",")

    if not folder_id or not profile_id:
        sys.exit("Set MLX_FOLDER_ID and MLX_PROFILE_ID")

    token = token_from_env()
    results = []

    with sync_playwright() as pw:
        port = start_profile(token, folder_id, profile_id, automation_type="playwright", headless=True)
        browser = pw.chromium.connect_over_cdp(f"http://127.0.0.1:{port}")
        page = browser.contexts[0].new_page()

        for url in urls:
            url = url.strip()
            if not url:
                continue
            page.goto(url, timeout=30000)
            results.append({"url": url, "title": page.title()})

        page.close()
        stop_profile(token, profile_id)

    print(json.dumps({"pages": len(results), "results": results}, indent=2))