Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-engine

Source of release 0.6.1

0.6.1 / tools / scroll_bench / save_page.py

#!/usr/bin/env python3
"""Save a web page with its style sheets, fonts and images, for scroll_bench.

    python3 -I tools/scroll_bench/save_page.py URL DIRECTORY

Writes DIRECTORY/index.html and DIRECTORY/res/*, with the page's references to style sheets,
images (src, srcset's first candidate, poster) and the url()s of the style sheets rewritten to
the saved copies. Scripts are dropped (the engine runs none yet). Real pages stay out of the
repository: save them in a scratch directory and point scroll_bench at them.
"""
import hashlib
import html.parser
import os
import re
import sys
import urllib.parse
import urllib.request

AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
LIMIT = 8 * 1024 * 1024


def fetch(url):
    request = urllib.request.Request(url, headers={"User-Agent": AGENT, "Accept-Encoding": "identity"})
    with urllib.request.urlopen(request, timeout=30) as response:
        return response.read(LIMIT), response.headers.get_content_charset() or "utf-8"


class Saver:
    def __init__(self, directory):
        self.directory = directory
        self.saved = {}
        os.makedirs(os.path.join(directory, "res"), exist_ok=True)

    def save(self, url, css=False):
        """The local path (relative to index.html) of url's saved copy, or url on failure."""
        if url in self.saved:
            return self.saved[url]
        if not url.startswith(("http://", "https://")):
            return url
        path = urllib.parse.urlparse(url).path
        extension = os.path.splitext(path)[1][:8] or (".css" if css else "")
        name = "res/" + hashlib.sha1(url.encode()).hexdigest()[:16] + extension
        self.saved[url] = name
        try:
            data, charset = fetch(url)
        except Exception as failure:  # a missing resource leaves its reference as it was
            print(f"save_page: {url}: {failure}", file=sys.stderr)
            self.saved[url] = url
            return url
        if css:
            text = data.decode(charset, errors="replace")
            data = self.rewrite_css(text, url, in_sheet=True).encode("utf-8")
        with open(os.path.join(self.directory, name), "wb") as file:
            file.write(data)
        return name

    def rewrite_css(self, text, base, in_sheet):
        def replace_url(match):
            reference = match.group(2).strip()
            if reference.startswith("data:") or not reference:
                return match.group(0)
            local = self.save(urllib.parse.urljoin(base, reference))
            # A style sheet's url()s resolve against the sheet itself (res/), the page's
            # against index.html.
            local = local[4:] if local.startswith("res/") and in_sheet else local
            return f'url("{local}")'

        def replace_import(match):
            local = self.save(urllib.parse.urljoin(base, match.group(2)), css=True)
            local = local[4:] if local.startswith("res/") and in_sheet else local
            return f'@import "{local}"'

        text = re.sub(r'@import\s+(["\'])([^"\']+)\1', replace_import, text)
        return re.sub(r'url\(\s*(["\']?)([^"\')]+)\1\s*\)', replace_url, text)


class Rewriter(html.parser.HTMLParser):
    def __init__(self, saver, base):
        super().__init__(convert_charrefs=False)
        self.saver = saver
        self.base = base
        self.out = []
        self.in_script = False
        self.in_style = False

    def handle_starttag(self, tag, attributes):
        self.out.append(self.tag_text(tag, attributes, close=False))

    def handle_startendtag(self, tag, attributes):
        self.out.append(self.tag_text(tag, attributes, close=True))

    def tag_text(self, tag, attributes, close):
        values = dict(attributes)
        if tag == "base" and values.get("href"):
            self.base = urllib.parse.urljoin(self.base, values["href"])
            return ""
        if tag == "script":
            self.in_script = True
            return ""
        if tag == "style":
            self.in_style = True
        kept = []
        for name, value in attributes:
            if value is None:
                kept.append(name)
                continue
            if name.startswith("on") or name in ("integrity", "crossorigin"):
                continue
            # Media is not played by the engine yet: a video or audio source is dropped (its
            # poster is kept).
            if name == "src" and tag in ("video", "audio", "source"):
                continue
            if tag == "link" and name == "href" and "stylesheet" in (values.get("rel") or ""):
                value = self.saver.save(urllib.parse.urljoin(self.base, value), css=True)
            elif tag == "link" and name == "href" and (values.get("rel") or "") in ("icon", "shortcut icon"):
                value = self.saver.save(urllib.parse.urljoin(self.base, value))
            elif name in ("src", "poster") and tag in ("img", "video", "input"):
                value = self.saver.save(urllib.parse.urljoin(self.base, value))
            elif name == "srcset":
                first = value.split(",")[0].strip().split(" ")[0]
                value = self.saver.save(urllib.parse.urljoin(self.base, first))
            elif name == "style":
                value = self.saver.rewrite_css(value, self.base, in_sheet=False)
            kept.append(f'{name}="{html.escape(value, quote=True)}"')
        text = "<" + " ".join([tag] + kept)
        return text + (" />" if close else ">")

    def handle_endtag(self, tag):
        if tag == "script":
            self.in_script = False
            return
        if tag == "style":
            self.in_style = False
        self.out.append(f"</{tag}>")

    def handle_data(self, data):
        if self.in_script:
            return
        if self.in_style:
            data = self.saver.rewrite_css(data, self.base, in_sheet=False)
        self.out.append(data)

    def handle_entityref(self, name):
        self.out.append(f"&{name};")

    def handle_charref(self, name):
        self.out.append(f"&#{name};")

    def handle_comment(self, data):
        pass

    def handle_decl(self, decl):
        self.out.append(f"<!{decl}>")


def main():
    if len(sys.argv) != 3:
        print(__doc__, file=sys.stderr)
        return 2
    url, directory = sys.argv[1], sys.argv[2]
    data, charset = fetch(url)
    saver = Saver(directory)
    rewriter = Rewriter(saver, url)
    rewriter.feed(data.decode(charset, errors="replace"))
    rewriter.close()
    with open(os.path.join(directory, "index.html"), "w", encoding="utf-8") as file:
        file.write("".join(rewriter.out))
    print(f"save_page: {url} -> {directory} ({len(saver.saved)} resources)")
    return 0


if __name__ == "__main__":
    sys.exit(main())