advanced_clone_spa_gui.py

import os
import re
import requests
from urllib.parse import urljoin, urlparse
from bs4 import BeautifulSoup
import tkinter as tk
from tkinter import ttk, filedialog, messagebox, scrolledtext
import threading
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options

class WebClonerSPA:
    def __init__(self, root):
        self.root = root
        self.root.title("Advanced Web Cloner (SPA Support)")
        self.root.geometry("900x700")

        # Variabel GUI
        self.target_url = tk.StringVar()
        self.output_dir = tk.StringVar(value="cloned_site")
        self.max_depth = tk.IntVar(value=2)
        self.is_running = False

        # Setup GUI
        self.setup_gui()

    def setup_gui(self):
        # Frame input
        input_frame = ttk.Frame(self.root, padding="10")
        input_frame.grid(row=0, column=0, sticky="ew")
        self.root.columnconfigure(0, weight=1)
        self.root.rowconfigure(1, weight=1)

        ttk.Label(input_frame, text="URL Target:").grid(row=0, column=0, sticky="w", pady=2)
        ttk.Entry(input_frame, textvariable=self.target_url, width=70).grid(row=0, column=1, padx=5, pady=2)
        ttk.Button(input_frame, text="Pilih Folder", command=self.select_output_dir).grid(row=0, column=2, padx=5)

        ttk.Label(input_frame, text="Output Folder:").grid(row=1, column=0, sticky="w", pady=2)
        ttk.Entry(input_frame, textvariable=self.output_dir, width=70).grid(row=1, column=1, padx=5, pady=2)

        ttk.Label(input_frame, text="Max Kedalaman (rekursif):").grid(row=2, column=0, sticky="w", pady=2)
        ttk.Spinbox(input_frame, from_=1, to=5, textvariable=self.max_depth, width=5).grid(row=2, column=1, sticky="w", padx=5, pady=2)

        # Tombol
        btn_frame = ttk.Frame(self.root, padding="10")
        btn_frame.grid(row=1, column=0, sticky="nsew")
        self.start_btn = ttk.Button(btn_frame, text="Mulai Clone", command=self.start_clone)
        self.start_btn.grid(row=0, column=0, padx=5)

        # Log area
        self.log_text = scrolledtext.ScrolledText(btn_frame, wrap=tk.WORD, state='disabled')
        self.log_text.grid(row=1, column=0, pady=10, sticky="nsew")
        btn_frame.columnconfigure(0, weight=1)
        btn_frame.rowconfigure(1, weight=1)

    def select_output_dir(self):
        folder = filedialog.askdirectory()
        if folder:
            self.output_dir.set(folder)

    def log(self, msg):
        self.log_text.config(state='normal')
        self.log_text.insert(tk.END, msg + "\n")
        self.log_text.see(tk.END)
        self.log_text.config(state='disabled')
        self.root.update_idletasks()

    def is_valid(self, url):
        parsed = urlparse(url)
        return bool(parsed.netloc) and bool(parsed.scheme)

    def fix_resource_paths(self, soup, base_url, output_dir):
        base_netloc = urlparse(base_url).netloc

        # CSS
        for tag in soup.find_all("link", rel="stylesheet"):
            href = tag.get("href")
            if href:
                full_url = urljoin(base_url, href)
                if urlparse(full_url).netloc == base_netloc:
                    parsed = urlparse(full_url)
                    rel_path = parsed.path.lstrip("/")
                    if not rel_path or rel_path.endswith("/"):
                        rel_path = os.path.join(rel_path, "style.css")
                    local_path = os.path.relpath(os.path.join(output_dir, rel_path), output_dir)
                    tag["href"] = local_path

        # JS
        for tag in soup.find_all("script", src=True):
            src = tag.get("src")
            if src:
                full_url = urljoin(base_url, src)
                if urlparse(full_url).netloc == base_netloc:
                    parsed = urlparse(full_url)
                    rel_path = parsed.path.lstrip("/")
                    if not rel_path or rel_path.endswith("/"):
                        rel_path = os.path.join(rel_path, "script.js")
                    local_path = os.path.relpath(os.path.join(output_dir, rel_path), output_dir)
                    tag["src"] = local_path

        # Images
        for tag in soup.find_all("img", src=True):
            src = tag.get("src")
            if src:
                full_url = urljoin(base_url, src)
                if urlparse(full_url).netloc == base_netloc:
                    parsed = urlparse(full_url)
                    rel_path = parsed.path.lstrip("/")
                    local_path = os.path.relpath(os.path.join(output_dir, rel_path), output_dir)
                    tag["src"] = local_path

    def get_all_links(self, soup, base_url):
        links = set()
        for a_tag in soup.find_all("a", href=True):
            href = a_tag["href"]
            full_url = urljoin(base_url, href)
            if self.is_valid(full_url) and urlparse(full_url).netloc == urlparse(base_url).netloc:
                links.add(full_url)
        return links

    def get_all_resources(self, soup, base_url):
        resources = set()

        # CSS
        for link in soup.find_all("link", rel="stylesheet"):
            href = link.get("href")
            if href:
                full_url = urljoin(base_url, href)
                if self.is_valid(full_url) and urlparse(full_url).netloc == urlparse(base_url).netloc:
                    resources.add(full_url)

        # JS
        for script in soup.find_all("script", src=True):
            src = script.get("src")
            if src:
                full_url = urljoin(base_url, src)
                if self.is_valid(full_url) and urlparse(full_url).netloc == urlparse(base_url).netloc:
                    resources.add(full_url)

        # Images
        for img in soup.find_all("img", src=True):
            src = img.get("src")
            if src:
                full_url = urljoin(base_url, src)
                if self.is_valid(full_url) and urlparse(full_url).netloc == urlparse(base_url).netloc:
                    resources.add(full_url)

        return resources

    def download_file(self, url, folder):
        try:
            response = requests.get(url, timeout=10)
            response.raise_for_status()

            parsed = urlparse(url)
            rel_path = parsed.path.lstrip("/")
            if not rel_path or rel_path.endswith("/"):
                rel_path = os.path.join(rel_path, "index.html")

            full_path = os.path.join(folder, rel_path)
            os.makedirs(os.path.dirname(full_path), exist_ok=True)

            with open(full_path, "wb") as f:
                f.write(response.content)
            self.log(f"Downloaded: {url}")
        except Exception as e:
            self.log(f"Failed to download {url}: {e}")

    def clone_page(self, url, folder, visited, depth, max_depth):
        if depth > max_depth or url in visited:
            return
        visited.add(url)

        try:
            # Setup Chrome untuk Selenium
            options = Options()
            options.add_argument("--headless")  # Nonaktifkan GUI browser
            options.add_argument("--no-sandbox")
            options.add_argument("--disable-dev-shm-usage")

            # Ganti path ke chromedriver jika tidak di PATH
            # service = Service("/path/to/chromedriver")
            # driver = webdriver.Chrome(service=service, options=options)
            driver = webdriver.Chrome(options=options)

            driver.get(url)
            # Tunggu sedikit agar JS selesai load
            driver.implicitly_wait(5)

            html = driver.page_source
            driver.quit()

            soup = BeautifulSoup(html, "html.parser")

            # Fix path lokal sebelum menyimpan
            self.fix_resource_paths(soup, url, folder)

            # Simpan HTML
            parsed = urlparse(url)
            rel_path = parsed.path.lstrip("/")
            if not rel_path or rel_path.endswith("/"):
                rel_path = os.path.join(rel_path, "index.html")
            full_path = os.path.join(folder, rel_path)
            os.makedirs(os.path.dirname(full_path), exist_ok=True)

            with open(full_path, "w", encoding="utf-8") as f:
                f.write(str(soup))

            self.log(f"Saved: {url}")

            # Ambil resource
            resources = self.get_all_resources(soup, url)
            for res_url in resources:
                self.download_file(res_url, folder)

            # Ambil link untuk rekursif
            if depth < max_depth:
                links = self.get_all_links(soup, url)
                for link in links:
                    self.clone_page(link, folder, visited, depth + 1, max_depth)

        except Exception as e:
            self.log(f"Error cloning {url}: {e}")

    def start_clone(self):
        if self.is_running:
            return
        self.is_running = True
        self.start_btn.config(state='disabled')
        threading.Thread(target=self.run_clone, daemon=True).start()

    def run_clone(self):
        try:
            url = self.target_url.get().strip()
            folder = self.output_dir.get()
            max_depth = self.max_depth.get()

            if not url:
                messagebox.showerror("Error", "URL tidak boleh kosong!")
                return

            self.log("Mulai cloning (SPA support)...")
            visited = set()
            self.clone_page(url, folder, visited, 1, max_depth)
            self.log("Cloning selesai!")
        except Exception as e:
            self.log(f"Error: {e}")
        finally:
            self.is_running = False
            self.start_btn.config(state='normal')

if __name__ == "__main__":
    root = tk.Tk()
    app = WebClonerSPA(root)
    root.mainloop()