main.py

#!/usr/bin/env python3
"""Script created by opencode:arc/apex

Scrapes the "Making History: Shakespeare and the Royal Family" exhibition at
https://sharc.kcl.ac.uk/exhibition/ into a portable, self-contained static
HTML mirror under ./html/ that works fully offline (open via file:// or a
local `python -m http.server`).

Usage:
    python main.py [url] [output_dir]
"""

import sys
import os
import re
import json
import time
import posixpath
import urllib.parse
from collections import deque, OrderedDict

import requests
from bs4 import BeautifulSoup

# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------

SITE_URL = "https://sharc.kcl.ac.uk/exhibition/"
HOST = "sharc.kcl.ac.uk"
DEFAULT_OUTPUT_DIR = "html"
ASSETS_PREFIX = "_assets"

USER_AGENT = (
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
ACCEPT_HTML = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
ACCEPT_IMAGE = "image/avif,image/webp,image/apng,image/*,*/*;q=0.8"
ACCEPT_CSS = "text/css,*/*;q=0.1"
ACCEPT_JSON = "application/json,application/ld+json,*/*;q=0.1"

TIMEOUT = 45
RETRIES = 2
BACKOFF = 1.5
MAX_WORKERS = 8

IIIF_HOSTS = ("rct.resourcespace.com", "images.cogapp.com")
IIIF_HIRES_SIZES = ("full/2000,/0/default.jpg", "full/1024,1024/0/default.jpg", "full/600,/0/default.jpg")

OPENSEADRAGON_URL = "https://cdn.jsdelivr.net/npm/openseadragon@2.4.2/build/openseadragon/openseadragon.min.js"
FA_CSS_URL = "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.4/css/all.min.css"
FA_WEBFONT_BASE = "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.4/webfonts/"
FONTS_CSS_URL = (
    "https://fonts.googleapis.com/css2?family=Open+Sans:ital,wght@0,400;0,600;"
    "0,700;0,800;1,400;1,600;1,700&family=Raleway:ital,wght@0,700;1,700&display=swap"
)

YT_POSTER_URL = "https://img.youtube.com/vi/{id}/hqdefault.jpg"

# Magic-byte sniffing for binary validation (servers here sometimes lie about
# content-type, returning an HTML error page with a 200 status).
JPEG_MAGIC = b"\xff\xd8\xff"
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
GIF_MAGICS = (b"GIF87a", b"GIF89a")
WEBP_MAGIC = b"RIFF"
WOFF2_MAGIC = b"wOF2"
WOFF_MAGIC = b"wOFF"
ICO_MAGIC = b"\x00\x00\x01\x00"
CUR_MAGIC = b"\x00\x00\x02\x00"

# Vendor/offline JavaScript injected into every page.
STUBS_JS = (
    "window.dataLayer=[];window.gtag=function(){window.dataLayer.push(arguments);};"
    "window.cookieconsent={initialise:function(){},hasInitialised:true};"
)

OFFLINE_VIEWER_JS = r"""/* Offline IIIF viewer replacement - generated by site2static (opencode:arc/apex).
   Replaces the live OpenSeadragon+info.json deep-zoom with a single high-res
   derivative served locally, reusing the on-page SVG icon symbols and the
   site's existing iiif-viewer CSS. */
document.addEventListener("DOMContentLoaded", function () {
  if (typeof OpenSeadragon === "undefined") { return; }
  var nodes = document.querySelectorAll("[data-iiif-offline]");
  var svgns = "http://www.w3.org/2000/svg";
  var xlinkns = "http://www.w3.org/1999/xlink";
  function makeButton(id, iconId, label) {
    var b = document.createElement("button");
    b.type = "button"; b.id = id; b.className = "iiif-viewer__button";
    b.setAttribute("aria-label", label); b.setAttribute("title", label);
    var svg = document.createElementNS(svgns, "svg");
    svg.setAttribute("viewBox", "0 0 24 24");
    svg.setAttribute("width", "24"); svg.setAttribute("height", "24");
    svg.setAttribute("fill", "currentColor"); svg.setAttribute("aria-hidden", "true");
    var use = document.createElementNS(svgns, "use");
    use.setAttributeNS(xlinkns, "href", "#" + iconId);
    svg.appendChild(use); b.appendChild(svg);
    return b;
  }
  for (var i = 0; i < nodes.length; i++) {
    (function () {
      var el = nodes[i];
      var id = el.getAttribute("data-iiif-id");
      var src = el.getAttribute("data-iiif-src");
      var mount = document.getElementById("osd-viewer-" + id);
      if (!mount || !src) { return; }
      var controls = document.createElement("div");
      controls.id = "osd-controls-" + id;
      controls.className = "iiif-viewer__controls";
      controls.appendChild(makeButton("osd-btn-zoom-in-" + id, "icon-zoom-in-" + id, "Zoom in"));
      controls.appendChild(makeButton("osd-btn-zoom-out-" + id, "icon-zoom-out-" + id, "Zoom out"));
      controls.appendChild(makeButton("osd-btn-full-page-" + id, "icon-fullscreen-" + id, "Toggle fullscreen"));
      mount.appendChild(controls);
      OpenSeadragon({
        id: "osd-viewer-" + id,
        tileSources: { type: "image", url: src },
        sequenceMode: false, showSequenceControl: false,
        showHomeControl: false, showNavigationControl: false,
        zoomInButton: "osd-btn-zoom-in-" + id,
        zoomOutButton: "osd-btn-zoom-out-" + id,
        fullPageButton: "osd-btn-full-page-" + id
      }).addHandler("open", function (e) {
        var icons = document.getElementById("osd-viewer-icons-" + id);
        if (icons) { e.eventSource.element.appendChild(icons.cloneNode(true)); }
      });
    })();
  }
});
"""

ASSET_EXTS = (
    ".css", ".js", ".svg", ".jpg", ".jpeg", ".png", ".gif", ".webp", ".ico",
    ".woff", ".woff2", ".ttf", ".otf", ".eot", ".webmanifest", ".xml", ".json",
)


# ---------------------------------------------------------------------------
# URL / path helpers
# ---------------------------------------------------------------------------

def fix_site_asset_url(url: str) -> str:
    """The live site references some assets as root-relative `/assets/...` which
    404; the real files live under `/exhibition/assets/...`. Return the working
    absolute URL for download while keeping the original literal for rewriting.
    """
    parsed = urllib.parse.urlsplit(url)
    if parsed.netloc == HOST and parsed.path.startswith("/assets/"):
        return "https://" + HOST + "/exhibition" + parsed.path
    return url


def is_same_host(url: str) -> bool:
    return urllib.parse.urlsplit(url).netloc == HOST


def is_iiif(url: str) -> bool:
    return urllib.parse.urlsplit(url).netloc in IIIF_HOSTS and "/iiif/" in url


def is_iiif_base(url: str) -> bool:
    return is_iiif(url) and "/full/" not in url and "/info.json" not in url


def page_key(url: str):
    """Return the canonical path key for an exhibition page, or None if the URL
    is not a crawlable page (external, asset, or outside /exhibition).
    """
    parsed = urllib.parse.urlsplit(url)
    if parsed.netloc != HOST:
        return None
    path = parsed.path or "/"
    if path in ("/exhibition", "/exhibition/"):
        return "/exhibition"
    if not path.startswith("/exhibition/"):
        return None
    rest = path[len("/exhibition/"):]
    if not rest:
        return "/exhibition"
    last = rest.rstrip("/").rsplit("/", 1)[-1]
    if "." in last and last.rsplit(".", 1)[-1].lower() in (
        "css", "js", "svg", "jpg", "jpeg", "png", "gif", "webp", "ico",
        "woff", "woff2", "ttf", "otf", "eot", "webmanifest", "xml", "json",
    ):
        return None
    return "/exhibition/" + rest.rstrip("/")


def page_file_for(key: str) -> str:
    if key == "/exhibition":
        return "index.html"
    rest = key[len("/exhibition/"):]
    return rest + "/index.html"


def relbase_of(page_file: str) -> str:
    return "../" * page_file.count("/")


def page_link(page_file: str, target_file: str) -> str:
    return posixpath.relpath(target_file, posixpath.dirname(page_file))


def safe_slug(s: str) -> str:
    s = urllib.parse.unquote(s)
    s = re.sub(r"[^A-Za-z0-9._-]+", "_", s)
    s = re.sub(r"_+", "_", s).strip("_.")
    return s[:140] or "x"


def ext_of(url: str) -> str:
    path = urllib.parse.urlsplit(url).path
    return os.path.splitext(path)[1].lower()


def iiif_local_path(url: str) -> str:
    parsed = urllib.parse.urlsplit(url)
    host = parsed.netloc
    after = parsed.path.split("/iiif/", 1)[1] if "/iiif/" in parsed.path else parsed.path.lstrip("/")
    return posixpath.join(ASSETS_PREFIX, "iiif", host, safe_slug(after) + ".jpg")


def iiif_hires_local_path(base: str) -> str:
    parsed = urllib.parse.urlsplit(base)
    host = parsed.netloc
    after = parsed.path.split("/iiif/", 1)[1] if "/iiif/" in parsed.path else parsed.path.lstrip("/")
    return posixpath.join(ASSETS_PREFIX, "iiif", host, safe_slug(after) + "_hires.jpg")


def site_asset_local(url: str) -> str:
    """Map a sharc.kcl.ac.uk/exhibition/<subpath> asset to a local _assets path."""
    parsed = urllib.parse.urlsplit(fix_site_asset_url(url))
    path = parsed.path
    if path.startswith("/exhibition/assets/"):
        rest = path[len("/exhibition/assets/"):]
        return posixpath.join(ASSETS_PREFIX, *rest.split("/"))
    if path.startswith("/exhibition/favicon/"):
        rest = path[len("/exhibition/favicon/"):]
        return posixpath.join(ASSETS_PREFIX, "favicon", *rest.split("/"))
    if path.startswith("/exhibition/assets/svg/"):
        rest = path[len("/exhibition/assets/svg/"):]
        return posixpath.join(ASSETS_PREFIX, "svg", rest)
    if path == "/exhibition/full/600,/0/default.jpg":
        return posixpath.join(ASSETS_PREFIX, "iiif", "broken-og", "sharc-og.jpg")
    return posixpath.join(ASSETS_PREFIX, "misc", safe_slug(path))


# ---------------------------------------------------------------------------
# HTTP layer
# ---------------------------------------------------------------------------

class Fetcher:
    def __init__(self):
        self.session = requests.Session()
        self.session.headers.update({"User-Agent": USER_AGENT})

    def get(self, url: str, accept: str = "*/*") -> requests.Response:
        last = None
        for attempt in range(RETRIES + 1):
            try:
                r = self.session.get(
                    url, headers={"Accept": accept, "Referer": SITE_URL},
                    timeout=TIMEOUT, allow_redirects=True,
                )
                return r
            except requests.RequestException as e:
                last = e
                time.sleep(BACKOFF * (attempt + 1))
        raise last


def sniff(data: bytes):
    if data.startswith(JPEG_MAGIC):
        return "image/jpeg"
    if data.startswith(PNG_MAGIC):
        return "image/png"
    if data.startswith(GIF_MAGICS):
        return "image/gif"
    if data[:4] == WEBP_MAGIC and data[8:12] == b"WEBP":
        return "image/webp"
    if data[:4] in (ICO_MAGIC, CUR_MAGIC):
        return "image/x-icon"
    if data[:4] in (WOFF2_MAGIC, WOFF_MAGIC):
        return "font/woff2"
    head = data.lstrip()[:300]
    if head[:5] == b"<?xml" or head[:4] == b"<svg" or (b"<svg" in head[:200] and b"<!doctype html" not in head[:200].lower()):
        return "image/svg+xml"
    return None


def validate(data: bytes, expected: str) -> bool:
    if not data:
        return False
    low = data[:200].lower()
    is_html_err = b"<!doctype html" in low or b"<html" in low[:40]
    if expected == "image":
        kind = sniff(data)
        return kind is not None and kind.startswith("image")
    if expected == "font":
        return data[:4] in (WOFF2_MAGIC, WOFF_MAGIC)
    if expected == "css":
        return not is_html_err and b"{" in data[:4000]
    if expected == "js":
        return not is_html_err
    if expected == "json":
        return not is_html_err and data.lstrip()[:1] in (b"{", b"[")
    if expected == "xml":
        return not is_html_err and data.lstrip()[:1] in (b"<",)
    if expected == "html":
        return True
    return not is_html_err


def expected_for(url: str) -> str:
    ext = ext_of(url)
    if ext in (".jpg", ".jpeg", ".png", ".gif", ".webp"):
        return "image"
    if ext == ".svg":
        return "image"
    if ext == ".ico":
        return "image"
    if ext in (".woff2", ".woff"):
        return "font"
    if ext == ".css":
        return "css"
    if ext == ".js":
        return "js"
    if ext in (".webmanifest", ".json"):
        return "json"
    if ext == ".xml":
        return "xml"
    if is_iiif(url):
        return "image"
    return "binary"


def accept_for(expected: str) -> str:
    return {
        "image": ACCEPT_IMAGE, "css": ACCEPT_CSS, "json": ACCEPT_JSON,
        "html": ACCEPT_HTML,
    }.get(expected, "*/*")


# ---------------------------------------------------------------------------
# Mirror state
# ---------------------------------------------------------------------------

class Mirror:
    def __init__(self, output_dir):
        self.output_dir = output_dir
        self.pages = OrderedDict()          # page_key -> raw html bytes
        self.page_files = {}                # page_key -> local file path
        self.page_links = {}                # page_key -> {literal: target_key}
        self.downloads = OrderedDict()      # download_url -> (expected, local)
        self.literal_to_url = {}            # literal -> download_url
        self.iiif_bases = OrderedDict()      # base_literal -> (base_url, img_id)
        self.iframes = {}                   # page_key -> [(kind, id)]
        self.ok_downloads = {}              # download_url -> local
        self.failed = []                     # [{url, reason}]
        self.asset_map = {}                 # literal -> local (successful)
        self.base_to_local = {}             # base_literal -> hires local
        self.video_posters = set()          # youtube ids with a downloaded poster

    def fs_path(self, rel: str) -> str:
        return os.path.join(self.output_dir, *rel.split("/"))


# ---------------------------------------------------------------------------
# Crawl
# ---------------------------------------------------------------------------

def collect_page(mirror, fetcher, key, url):
    ret = None
    try:
        r = fetcher.get(url, ACCEPT_HTML)
    except Exception as e:
        print("  ! page fetch failed:", url, e)
        mirror.failed.append({"url": url, "reason": "fetch error: %s" % e})
        return
    if r.status_code != 200 or "text/html" not in r.headers.get("Content-Type", ""):
        print("  ! page not html:", url, r.status_code)
        mirror.failed.append({"url": url, "reason": "non-html status %s" % r.status_code})
        return
    html = r.content
    mirror.pages[key] = html
    soup = BeautifulSoup(html, "html.parser")

    links = {}
    for a in soup.find_all("a", href=True):
        literal = a["href"]
        absurl = urllib.parse.urljoin(url, literal)
        tk = page_key(absurl)
        if tk is not None:
            links[literal] = tk
    for link in soup.find_all("link", href=True):
        rel = link.get("rel") or []
        if "home" in rel:
            literal = link["href"]
            absurl = urllib.parse.urljoin(url, literal)
            tk = page_key(absurl)
            if tk is not None:
                links[literal] = tk
    mirror.page_links[key] = links

    def add_asset(literal, download_url, local, expected=None):
        du = download_url
        if expected is None:
            expected = expected_for(du)
        mirror.literal_to_url[literal] = du
        if du not in mirror.downloads:
            mirror.downloads[du] = (expected, local)

    for link in soup.find_all("link", href=True):
        literal = link["href"]
        absurl = urllib.parse.urljoin(url, literal)
        if is_same_host(absurl) or is_iiif(absurl):
            add_asset(literal, fix_site_asset_url(absurl), site_asset_local(absurl))

    for s in soup.find_all("script", src=True):
        literal = s["src"]
        absurl = urllib.parse.urljoin(url, literal)
        if is_same_host(absurl):
            add_asset(literal, fix_site_asset_url(absurl), site_asset_local(absurl))

    for img in soup.find_all("img", src=True):
        literal = img["src"]
        absurl = urllib.parse.urljoin(url, literal)
        if is_iiif(absurl):
            add_asset(literal, absurl, iiif_local_path(absurl))
        elif is_same_host(absurl):
            add_asset(literal, fix_site_asset_url(absurl), site_asset_local(absurl))

    for v in soup.find_all(attrs={"data-img-source": True}):
        base = v["data-img-source"]
        base_abs = urllib.parse.urljoin(url, base).rstrip("/")
        img_id = v.get("data-img-id", "")
        mirror.iiif_bases[base] = (base_abs, img_id)

    for m in soup.find_all("meta"):
        prop = m.get("property") or m.get("name")
        if prop in ("og:image", "twitter:image"):
            content = m.get("content", "")
            if not content:
                continue
            absurl = urllib.parse.urljoin(url, content)
            if is_iiif(absurl):
                add_asset(content, absurl, iiif_local_path(absurl))
            elif is_same_host(absurl):
                add_asset(content, fix_site_asset_url(absurl), site_asset_local(absurl))

    for sc in soup.find_all("script", attrs={"type": "application/ld+json"}):
        txt = sc.string or ""
        for m in re.finditer(r'"url"\s*:\s*"(https?://[^"]+)"', txt):
            u = m.group(1)
            if is_iiif(u):
                add_asset(u, u, iiif_local_path(u))
            elif is_same_host(u) and "/full/" in u:
                add_asset(u, fix_site_asset_url(u), site_asset_local(u))

    iframes = []
    for ifr in soup.find_all("iframe", src=True):
        src = urllib.parse.urljoin(url, ifr["src"])
        ym = re.search(r"youtube\.com/embed/([A-Za-z0-9_-]+)", src)
        sm = re.search(r"sketchfab\.com/models/([A-Za-z0-9]+)/embed", src)
        if ym:
            iframes.append(("youtube", ym.group(1)))
        elif sm:
            iframes.append(("sketchfab", sm.group(1)))
    mirror.iframes[key] = iframes


def crawl(mirror, fetcher, start_url):
    start_key = page_key(start_url)
    seen = {start_key}
    queue = deque([start_key])
    mirror.page_files[start_key] = page_file_for(start_key)
    while queue:
        key = queue.popleft()
        url = "https://" + HOST + (key if key != "/exhibition" else "/exhibition") + "/"
        if key == "/exhibition":
            url = SITE_URL
        print("crawling", key)
        collect_page(mirror, fetcher, key, url)
        for literal, tk in mirror.page_links.get(key, {}).items():
            if tk not in seen:
                seen.add(tk)
                mirror.page_files[tk] = page_file_for(tk)
                queue.append(tk)


# ---------------------------------------------------------------------------
# Asset discovery (vendor CSS reveals further URLs)
# ---------------------------------------------------------------------------

def discover_vendor_assets(mirror, fetcher):
    # OpenSeadragon
    mirror.downloads[OPENSEADRAGON_URL] = ("js", posixpath.join(ASSETS_PREFIX, "js", "openseadragon.min.js"))

    # Font Awesome CSS + its webfonts
    r = fetcher.get(FA_CSS_URL, ACCEPT_CSS)
    fa_css = r.content
    validate_or_fail(mirror, FA_CSS_URL, fa_css, "css")
    fa_local = posixpath.join(ASSETS_PREFIX, "css", "fontawesome.min.css")
    mirror.ok_downloads[FA_CSS_URL] = fa_local
    mirror._fa_css = fa_css
    for m in re.finditer(r"url\(\.\./webfonts/([^)]+\.woff2?)\)", fa_css.decode("utf-8", "replace")):
        name = m.group(1)
        wu = FA_WEBFONT_BASE + name
        wl = posixpath.join(ASSETS_PREFIX, "webfonts", name)
        mirror.downloads[wu] = ("font", wl)

    # Google Fonts CSS + its woff2 files
    r = fetcher.get(FONTS_CSS_URL, ACCEPT_CSS)
    gf_css = r.content
    validate_or_fail(mirror, FONTS_CSS_URL, gf_css, "css")
    gf_local = posixpath.join(ASSETS_PREFIX, "css", "fonts.css")
    mirror.ok_downloads[FONTS_CSS_URL] = gf_local
    mirror._gf_css = gf_css
    for m in re.finditer(r"url\((https://fonts\.gstatic\.com/[^)]+\.woff2?)\)", gf_css.decode("utf-8", "replace")):
        wu = m.group(1)
        name = posixpath.basename(urllib.parse.urlsplit(wu).path)
        wl = posixpath.join(ASSETS_PREFIX, "fonts", name)
        mirror.downloads[wu] = ("font", wl)
        mirror._gf_rewrites = getattr(mirror, "_gf_rewrites", {})
        mirror._gf_rewrites[wu] = wl

    # IIIF high-res derivatives for every guided-tour viewer. Try sizes from
    # largest to smallest and keep the first that validates, so the viewer
    # still works on networks that refuse the largest size.
    for base_literal, (base_abs, _img_id) in mirror.iiif_bases.items():
        local = iiif_hires_local_path(base_abs)
        for size in IIIF_HIRES_SIZES:
            hire_url = base_abs + "/" + size
            try:
                r = fetcher.get(hire_url, ACCEPT_IMAGE)
                if r.status_code == 200 and validate(r.content, "image"):
                    os.makedirs(os.path.dirname(mirror.fs_path(local)), exist_ok=True)
                    with open(mirror.fs_path(local), "wb") as f:
                        f.write(r.content)
                    mirror.ok_downloads[hire_url] = local
                    mirror.base_to_local[base_literal] = local
                    print("  iiif-hires ok:", posixpath.basename(local), "<-", size)
                    break
            except Exception:
                pass

    # CSS background gradient svg (referenced as url(/assets/svg/gradient.svg))
    mirror.downloads["https://" + HOST + "/exhibition/assets/svg/gradient.svg"] = (
        "image", posixpath.join(ASSETS_PREFIX, "svg", "gradient.svg"))

    # YouTube poster images
    for key, ifs in mirror.iframes.items():
        for kind, vid in ifs:
            if kind == "youtube":
                mirror.downloads[YT_POSTER_URL.format(id=vid)] = (
                    "image", posixpath.join(ASSETS_PREFIX, "video", vid + ".jpg"))


def validate_or_fail(mirror, url, data, expected):
    if validate(data, expected):
        return True
    mirror.failed.append({"url": url, "reason": "validation failed (%s)" % expected})
    return False


# ---------------------------------------------------------------------------
# Downloading
# ---------------------------------------------------------------------------

def download_all(mirror, fetcher):
    from concurrent.futures import ThreadPoolExecutor, as_completed

    items = list(mirror.downloads.items())

    def work(item):
        url, (expected, local) = item
        try:
            r = fetcher.get(url, accept_for(expected))
        except Exception as e:
            return url, local, False, "fetch error: %s" % e
        if r.status_code != 200:
            return url, local, False, "status %s" % r.status_code
        if not validate(r.content, expected):
            return url, local, False, "validation (%s) ct=%s" % (expected, r.headers.get("Content-Type"))
        os.makedirs(os.path.dirname(mirror.fs_path(local)), exist_ok=True)
        with open(mirror.fs_path(local), "wb") as f:
            f.write(r.content)
        return url, local, True, None

    done = 0
    total = len(items)
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex:
        futs = [ex.submit(work, it) for it in items]
        for fut in as_completed(futs):
            url, local, ok, reason = fut.result()
            done += 1
            if ok:
                mirror.ok_downloads[url] = local
                print("  ok [%d/%d] %s" % (done, total, posixpath.basename(local)))
            else:
                mirror.failed.append({"url": url, "reason": reason})
                print("  !! [%d/%d] %s (%s)" % (done, total, url.split("/")[-1][:40], reason))

    # Build asset_map (literal -> local) for successful downloads
    for literal, du in mirror.literal_to_url.items():
        if du in mirror.ok_downloads:
            mirror.asset_map[literal] = mirror.ok_downloads[du]

    # Video posters
    for key, ifs in mirror.iframes.items():
        for kind, vid in ifs:
            if kind == "youtube":
                purl = YT_POSTER_URL.format(id=vid)
                if purl in mirror.ok_downloads:
                    mirror.video_posters.add(vid)


# ---------------------------------------------------------------------------
# CSS rewriting
# ---------------------------------------------------------------------------

def write_vendor_css(mirror):
    # main.css: rewrite the broken url(/assets/svg/gradient.svg) to local.
    css_url = "https://" + HOST + "/exhibition/assets/css/main.css"
    if css_url in mirror.ok_downloads:
        local = mirror.ok_downloads[css_url]
        with open(mirror.fs_path(local), "rb") as f:
            css = f.read().decode("utf-8", "replace")
        css = css.replace("url(/assets/svg/gradient.svg)", "url(../svg/gradient.svg)")
        with open(mirror.fs_path(local), "w", encoding="utf-8") as f:
            f.write(css)

    # Font Awesome CSS: keep its ../webfonts/ references (files stored there).
    if hasattr(mirror, "_fa_css"):
        with open(mirror.fs_path(posixpath.join(ASSETS_PREFIX, "css", "fontawesome.min.css")), "wb") as f:
            f.write(mirror._fa_css)

    # Google Fonts CSS: rewrite absolute woff2 URLs to local ../fonts/<name>.
    if hasattr(mirror, "_gf_css"):
        css = mirror._gf_css.decode("utf-8", "replace")
        for wu, wl in getattr(mirror, "_gf_rewrites", {}).items():
            css = css.replace("url(" + wu + ")", "url(../fonts/" + posixpath.basename(wl) + ")")
        with open(mirror.fs_path(posixpath.join(ASSETS_PREFIX, "css", "fonts.css")), "w", encoding="utf-8") as f:
            f.write(css)

    # Write the offline viewer script.
    with open(mirror.fs_path(posixpath.join(ASSETS_PREFIX, "js", "offline-viewer.js")), "w", encoding="utf-8") as f:
        f.write(OFFLINE_VIEWER_JS)


# ---------------------------------------------------------------------------
# HTML rewriting (byte-faithful, targeted)
# ---------------------------------------------------------------------------

GA_ASYNC_RE = re.compile(
    r'<script\s+async\s+src="https://www\.googletagmanager\.com/gtag/js\?id=[^"]*"></script>\s*', re.I)
GA_INLINE_RE = re.compile(
    r'<script>\s*window\.dataLayer\s*=\s*window\.dataLayer\s*\|\|\s*\[\];.*?'
    r"gtag\('config'[^)]*\);\s*</script>\s*", re.S | re.I)
CC_SCRIPT_RE = re.compile(
    r'<script\s+src="https://cdn\.jsdelivr\.net/npm/cookieconsent@3/build/cookieconsent\.min\.js"></script>\s*', re.I)
FONTS_LINK_RE = re.compile(r'href="https://fonts\.googleapis\.com/css2[^"]*"')


def rewrite_html(mirror, key, html_bytes):
    out = html_bytes.decode("utf-8")
    page_file = mirror.page_files[key]
    relbase = relbase_of(page_file)

    # 1. Strip analytics + cookieconsent CDN.
    out = GA_ASYNC_RE.sub("", out)
    out = GA_INLINE_RE.sub("", out)
    out = CC_SCRIPT_RE.sub("", out)

    # 2. Replace Font Awesome kit with self-hosted FA5 CSS.
    out = out.replace(
        '<script src="https://kit.fontawesome.com/2d4dd2d54f.js" crossorigin="anonymous"></script>',
        '<link rel="stylesheet" href="%s%s/fontawesome.min.css">' % (relbase, posixpath.join(ASSETS_PREFIX, "css")),
    )

    # 3. Localize Google Fonts <link>.
    out = FONTS_LINK_RE.sub('href="%s%s/fonts.css"' % (relbase, posixpath.join(ASSETS_PREFIX, "css")), out)

    # 4. Global asset URL replacement (longest first to avoid prefix issues).
    for literal in sorted(mirror.asset_map, key=len, reverse=True):
        local = mirror.asset_map[literal]
        out = out.replace(literal, relbase + local)

    # 5. Neutralise IIIF viewers whose hires image downloaded.
    for base_literal, hires_local in mirror.base_to_local.items():
        pattern = re.compile(
            r'class="iiif-viewer js-iiif-viewer" data-img-source="' +
            re.escape(base_literal) + r'" data-img-id="([0-9]+)"'
        )
        out = pattern.sub(
            lambda m: ('class="iiif-viewer" data-iiif-offline data-iiif-id="%s" '
                       'data-iiif-src="%s%s"' % (m.group(1), relbase, hires_local)),
            out,
        )

    # 6. Localise internal page links (<a href> and <link rel="home">).
    for literal, tk in mirror.page_links.get(key, {}).items():
        target_file = mirror.page_files[tk]
        local = page_link(page_file, target_file)
        out = re.sub(
            r'(href=(["\']))' + re.escape(literal) + r'\2',
            lambda m, loc=local: m.group(1) + loc + m.group(2),
            out,
        )

    # 7. Offline placeholders for YouTube / Sketchfab embeds.
    out = insert_embed_notes(out, key, mirror, relbase)

    # 8. Inject stubs + OpenSeadragon + offline-viewer around main.bundle.js.
    bundle_local = posixpath.join(ASSETS_PREFIX, "js", "main.bundle.js")
    bundle_tag = '<script src="%s%s"></script>' % (relbase, bundle_local)
    replacement = (
        '<script>%s</script>\n' % STUBS_JS +
        bundle_tag + '\n' +
        '<script src="%s%s/openseadragon.min.js"></script>\n' % (relbase, posixpath.join(ASSETS_PREFIX, "js")) +
        '<script src="%s%s/offline-viewer.js"></script>' % (relbase, posixpath.join(ASSETS_PREFIX, "js"))
    )
    out = out.replace(bundle_tag, replacement)

    return out.encode("utf-8")


def yt_note(vid, relbase, has_poster):
    img = ""
    if has_poster:
        img = ('<img src="' + relbase + ASSETS_PREFIX + '/video/' + vid + '.jpg" alt="Video thumbnail" '
               'style="display:block;width:100%;max-width:480px;margin-bottom:.75em;border:1px solid #ddd;" />')
    return (
        '<div class="offline-embed-note" '
        'style="margin:1.5em 0;padding:1em;background:#f4f4f4;border-left:4px solid #999;">'
        + img +
        '<p style="margin:0;font-size:.95rem;">This <strong>video</strong> is hosted on YouTube and '
        'plays in the embedded player below when you are online. '
        '<a href="https://www.youtube.com/watch?v=' + vid + '" target="_blank" rel="noopener">Open on YouTube</a>.</p>'
        '</div>'
    )


def sketchfab_note(mid):
    return (
        '<div class="offline-embed-note" '
        'style="margin:1.5em 0;padding:1em;background:#f4f4f4;border-left:4px solid #999;">'
        '<p style="margin:0;font-size:.95rem;">This <strong>3D model</strong> is hosted on Sketchfab and '
        'requires an internet connection to view in the embedded player below. '
        '<a href="https://sketchfab.com/3d-models/' + mid + '" target="_blank" rel="noopener">Open on Sketchfab</a>.</p>'
        '</div>'
    )


def insert_embed_notes(out, key, mirror, relbase):
    posters = mirror.video_posters

    def yt_repl(m):
        vid = m.group(2)
        return yt_note(vid, relbase, vid in posters) + m.group(1)

    out = re.sub(
        r'(<iframe\b[^>]*src="https://www\.youtube\.com/embed/([A-Za-z0-9_-]+)(?:\?[^"]*)?"[^>]*>\s*</iframe>)',
        yt_repl, out,
    )

    def sf_repl(m):
        return sketchfab_note(m.group(2)) + m.group(1)

    out = re.sub(
        r'(<iframe\b[^>]*src="https://sketchfab\.com/models/([A-Za-z0-9]+)/embed"[^>]*>\s*</iframe>)',
        sf_repl, out,
    )
    return out


# ---------------------------------------------------------------------------
# Output: pages, manifest, notes
# ---------------------------------------------------------------------------

def write_pages(mirror):
    for key, raw in mirror.pages.items():
        page_file = mirror.page_files[key]
        rewritten = rewrite_html(mirror, key, raw)
        path = mirror.fs_path(page_file)
        os.makedirs(os.path.dirname(path), exist_ok=True)
        with open(path, "wb") as f:
            f.write(rewritten)


def write_manifest(mirror):
    manifest = {
        "source": SITE_URL,
        "pages": len(mirror.pages),
        "assets_ok": len(mirror.ok_downloads),
        "assets_failed": len(mirror.failed),
        "failures": mirror.failed,
    }
    with open(mirror.fs_path("manifest.json"), "w", encoding="utf-8") as f:
        json.dump(manifest, f, indent=2)


NOTES_MD = """# Offline mirror notes

This folder is a static, self-contained copy of the *Making History:
Shakespeare and the Royal Family* exhibition at
<https://sharc.kcl.ac.uk/exhibition/>.

## Opening it

- Double-click `index.html` (opens via `file://`), **or**
- Run `python -m http.server` in this folder and open <http://localhost:8000/>.

All internal links and assets use depth-relative paths, so no server is
required.

## What works fully offline

- Every page, the navigation menu, hero carousel, breadcrumbs, and
  next/related-object links.
- The "Find out more about ..." contextual-object modals (pure in-page HTML,
  no network calls).
- All text, layout, CSS, fonts (Open Sans / Raleway), and icons
  (self-hosted Font Awesome 5).
- Images: every IIIF thumbnail, hero image, and object image that could be
- IIIF "Guided tour" deep-zoom viewer: replaced with a local high-resolution
  derivative served to a vendored copy of OpenSeadragon 2.4.2. You still get
  pan, zoom-in/out, and fullscreen. It is single-resolution (limited to the
  downloaded high-res image) rather than pixel-level deep-zoom, because
  mirroring the full tile pyramid for every image is impractical.
  downloaded is stored under `_assets/`.

## What needs an internet connection (and why)

These features are third-party embeds that cannot run from a local copy
because the media/viewer is hosted by a third party. The embed is kept intact
so it works when you are online, and a captioned placeholder explains it.

- **3D models** (Sketchfab `<iframe>`): the model and its WebGL viewer are
  hosted on sketchfab.com and cannot run without an internet connection.
- **Videos** (YouTube `<iframe>`): the video stream is served by YouTube and
  cannot play without an internet connection.

## Possible gaps

- Some IIIF images on `rct.resourcespace.com` are served only to certain
  regions/networks; if a derivative could not be fetched it is listed in
  `manifest.json` (under `failures`) and the original URL is left in place,
  so it still loads when you are online. Re-running the scraper from a
  network with access to the Royal Collection Trust IIIF server will fill
  these in.
- The `og:image` / Twitter card image for a few object pages points at a
  broken path on the live site itself; these are metadata-only and left as
  the original URL.
- Google Analytics and the cookie-consent banner were intentionally removed
  (no value offline); harmless stubs are injected so the site JavaScript
  initialises cleanly.

See `manifest.json` for the full list of successfully mirrored assets and any
failures.
"""


def write_notes(mirror):
    with open(mirror.fs_path("OFFLINE_NOTES.md"), "w", encoding="utf-8") as f:
        f.write(NOTES_MD)


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def main():
    url = sys.argv[1] if len(sys.argv) > 1 else SITE_URL
    output_dir = sys.argv[2] if len(sys.argv) > 2 else DEFAULT_OUTPUT_DIR

    fetcher = Fetcher()
    mirror = Mirror(output_dir)
    os.makedirs(output_dir, exist_ok=True)

    print("== crawling ==", url)
    crawl(mirror, fetcher, url)
    print("pages:", len(mirror.pages), "| iiif viewers:", len(mirror.iiif_bases),
          "| iframes:", sum(len(v) for v in mirror.iframes.values()))

    print("== discovering vendor assets ==")
    discover_vendor_assets(mirror, fetcher)
    print("assets to fetch:", len(mirror.downloads))

    print("== downloading assets ==")
    download_all(mirror, fetcher)

    print("== writing vendor css ==")
    write_vendor_css(mirror)

    print("== rewriting pages ==")
    write_pages(mirror)

    write_manifest(mirror)
    write_notes(mirror)

    print()
    print("done. pages:", len(mirror.pages),
          "| assets ok:", len(mirror.ok_downloads),
          "| failed:", len(mirror.failed))
    print("open:", os.path.join(output_dir, "index.html"))
    if mirror.failed:
        print("see", os.path.join(output_dir, "manifest.json"), "for failures")


if __name__ == "__main__":
    main()