Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-engine

Source of release 0.6.0

0.6.0 / tools / gen_dom_tree / module.lucb

#==============================================================================================
#
#   gen_dom_tree - the generator of the engine's HTML-template DOM trees (MediaControlsDOM)
#
#   DESCRIPTION:
#       Port of Meta/Generators/generate_dom_tree.py, which turns an HTML template into the
#       C++ constructor of a struct that builds that DOM under a parent node. It reads
#       data/html/MediaControls.html (and the style sheet its <link rel=stylesheet> names) and
#       writes src/web/generated/html/media_controls_dom.lucb, the Luce
#       constructor of HTML::MediaControlsDOM (the struct and its Options enum are in
#       generated/html/types_media_controls_dom.lucb).
#
#       As the Python generator does:
#         - `data-name` names an element: it becomes a field (a GC::Weak) of the struct;
#           other elements are `element_N` locals;
#         - `data-option="video"` wraps the element and its subtree in
#           `if (has_flag(options, Options::Video))`;
#         - inside <svg>, tag and attribute names resolve to SVG::TagNames / SVG::AttributeNames
#           when those declare them, else to the HTML tables;
#         - `<link rel=stylesheet href=...>` becomes a <style> element whose text is the
#           stripped style sheet, embedded as a constant;
#         - non-blank text sets the parent's text content; comments are dropped.
#       The Python generator reads the C++ TagNames.h / AttributeNames.h tables; this one reads
#       the engine's own declarations of them (html/types_tag_names.lucb, html/types_attribute_names.lucb,
#       svg/types_tag_names.lucb, svg/types_attribute_names.lucb) and answers Luce names. The
#       template is tokenized by a small scanner covering what Python's html.parser.HTMLParser
#       reports for it (start, end and self-closing tags, quoted attributes, text, comments).
#
#       Usage: gen_dom_tree DATA_DIRECTORY WEB_SOURCE_DIRECTORY OUTPUT_DIRECTORY, e.g.
#       gen_dom_tree data/html src/web src/web/generated/html.
#       test.sh regenerates into a temporary directory and compares with the committed fragment.
#
#       Copyright (c) 2026, the Ladybird developers.
#       Copyright (c) 2026 the luce-browser authors.
#       SPDX-License-Identifier: BSD-2-Clause (see data/html/LICENSE)
#
#==============================================================================================

import c
from luce_std import collections, files
import strings

## The HTML void elements, which cannot have children (HTML_VOID_ELEMENTS).
let html_void_elements: str[14] = ["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"]

## The most open elements the template may nest.
let max_depth: usize = 64

## An open element (DOMElement): its variable, tag and option.
struct OpenElement:
    var var_name: str
    var tag: str
    var option: str

## The generator's state (DOMTreeParser).
struct Generator:
    var output: collections.Buffer
    var tables: Tables
    var data_directory: str
    var indentation: usize
    var svg_depth: usize
    var unnamed_element_counter: usize
    var stack: OpenElement[64]
    var stack_size: usize
    var stylesheet_sources: collections.Buffer
    var stylesheet_count: usize
    ## The default member initializers of the named fields (`GC::Weak<DOM::Element> name;` starts null).
    var field_initializers: collections.Buffer
    ## The output's length before the last line, if that line was blank (`not self.lines[-1]`).
    var blank_line_start: usize?

## The four name tables, as the text of their Luce declarations.
struct Tables:
    var html_tags: str
    var html_attributes: str
    var svg_tags: str
    var svg_attributes: str

## Generate the MediaControlsDOM constructor from `arguments[1]` into `arguments[3]`.
pub func main(arguments: c.str[]) -> i32!:
    if arguments.length != 4:
        print("usage: gen_dom_tree DATA_DIRECTORY WEB_SOURCE_DIRECTORY OUTPUT_DIRECTORY")
        return 2
    let data_directory = try str(arguments[1])
    let web_directory = try str(arguments[2])
    let output_directory = try str(arguments[3])

    let tables = Tables(
        html_tags = try read_text(try c_cat([web_directory, "/html/types_tag_names.lucb"])),
        html_attributes = try read_text(try c_cat([web_directory, "/html/types_attribute_names.lucb"])),
        svg_tags = try read_text(try c_cat([web_directory, "/svg/types_tag_names.lucb"])),
        svg_attributes = try read_text(try c_cat([web_directory, "/svg/types_attribute_names.lucb"])),
    )
    let html = try read_text(try c_cat([data_directory, "/MediaControls.html"]))

    var generator: Generator = ---
    generator.output = collections.Buffer()
    generator.tables = tables
    generator.data_directory = data_directory
    generator.indentation = 0
    generator.svg_depth = 0
    generator.unnamed_element_counter = 0
    generator.stack_size = 0
    generator.stylesheet_sources = collections.Buffer()
    generator.stylesheet_count = 0
    generator.field_initializers = collections.Buffer()
    generator.blank_line_start = none
    try feed(&generator, html)

    # `if not parser.lines[-1]: parser.lines.pop()`
    if let start = generator.blank_line_start:
        generator.output.truncate(start)

    let fragment = try render(&generator)
    try files.write(try c_cat([output_directory, "/media_controls_dom.lucb"]), fragment.bytes)
    return 0

## The text of the file at `path`; the whole run keeps it, so it is never released.
func read_text(path: c.str) -> str!:
    let bytes = try files.read(path)
    return try str(bytes.value)

# mark: Output lines ============================================================================

## `_add_line(line)`: a line at the current indentation (one level is the function body's).
func add_line(this: Generator*, line: str) -> !:
    this.blank_line_start = none
    for _ in 0..<(this.indentation + 1):
        try this.output.put_text("    ")
    try this.output.put_text(line)
    try this.output.put_text("\n")

## `_add_line("")`: a blank line.
func add_blank_line(this: Generator*) -> !:
    this.blank_line_start = this.output.length()
    try this.output.put_text("\n")

## `_add_line(...)` of a formatted line.
func add_line_parts(this: Generator*, parts: const str[]) -> !:
    try add_line(this, try cat(parts))

# mark: Names ===================================================================================

## The Luce identifier of a template name: `-` and other non-identifier characters become `_`
## (to_snake_case) and an uppercase letter starts a new lowercase word (`viewBox` → `view_box`),
## which is how the engine's name tables spell the C++ identifiers.
func luce_identifier(name: str) -> str!:
    var builder: collections.Buffer
    for byte in name.bytes:
        if byte >= 'A' and byte <= 'Z':
            try builder.put_text("_")
            let lower: u8[1] = [byte + 32]
            _ = try builder.write(lower)
        elif (byte >= 'a' and byte <= 'z') or (byte >= '0' and byte <= '9') or byte == '_':
            let one: u8[1] = [byte]
            _ = try builder.write(one)
        else:
            try builder.put_text("_")
    return builder.text()

## Whether `table` declares `pub var <name>: ak.FlyString`.
func declares(table: str, name: str) -> bool!:
    let declaration = try cat(["\npub var ", name, ": ak.FlyString\n"])
    return strings.contains(table, declaration)

## resolve_tag: (namespace, tag name) of `tag`.
func resolve_tag(this: Generator*, tag: str, in_svg: bool) -> (str, str)!:
    let identifier = try luce_identifier(tag)
    let svg_name = try cat(["svg_tag_names_", identifier])
    if in_svg and try declares(this.tables.svg_tags, svg_name):
        return ("namespace_svg", svg_name)

    let html_name = try cat(["html_tag_names_", identifier])
    if not try declares(this.tables.html_tags, html_name):
        print(f"gen_dom_tree: unknown HTML tag {tag}")
        return error(error_unknown_name, "unknown HTML tag")
    return ("namespace_html", html_name)

## resolve_attribute: the attribute name of `attribute`.
func resolve_attribute(this: Generator*, attribute: str, in_svg: bool) -> str!:
    let identifier = try luce_identifier(attribute)
    let svg_name = try cat(["svg_attribute_names_", identifier])
    if in_svg and try declares(this.tables.svg_attributes, svg_name):
        return svg_name

    let html_name = try cat(["html_attribute_names_", identifier])
    if not try declares(this.tables.html_attributes, html_name):
        print(f"gen_dom_tree: unknown HTML attribute {attribute}")
        return error(error_unknown_name, "unknown HTML attribute")
    return html_name

## The error of a template name the tables do not declare.
let error_unknown_name: ErrorCode = ErrorCode.package(101)

## The error of a template the scanner cannot read.
let error_bad_template: ErrorCode = ErrorCode.package(102)

## A Luce string literal of `text` (`"..."` with `\\`, `\"`, `\n`, `\t` and `\r` escaped).
func quoted(text: str) -> str!:
    var builder: collections.Buffer
    try builder.put_text("\"")
    for byte in text.bytes:
        if byte == '\\':
            try builder.put_text("\\\\")
        elif byte == '"':
            try builder.put_text("\\\"")
        elif byte == '\n':
            try builder.put_text("\\n")
        elif byte == '\t':
            try builder.put_text("\\t")
        elif byte == '\r':
            try builder.put_text("\\r")
        else:
            let one: u8[1] = [byte]
            _ = try builder.write(one)
    try builder.put_text("\"")
    return builder.text()

# mark: Tags ====================================================================================

## `_current_parent()` as a DomNode* expression: `parent`, or the innermost open element's node.
func parent_node(this: Generator*) -> str!:
    if this.stack_size == 0:
        return "parent"
    return try cat([this.stack[this.stack_size - 1].var_name, ".dom_node()"])

## One attribute of a start tag.
struct Attribute:
    var name: str
    var value: str

## `_handle_tag(tag, attrs, self_closing)`.
func handle_tag(this: Generator*, tag: str, attributes: const Attribute[], self_closing: bool) -> !:
    # Wrap optional nodes in if statements
    var option = ""
    var data_name = ""
    for attribute in attributes:
        if attribute.name == "data-option":
            option = attribute.value
        elif attribute.name == "data-name":
            data_name = attribute.value
    if option != "":
        let option_case = try luce_identifier(option)
        try add_line_parts(this, ["if ((u8)options & (u8)HtmlMediaControlsDomOptions.", option_case, ") != 0:"])
        this.indentation += 1

    # Handle SVG contexts
    var in_svg = this.svg_depth > 0
    if tag == "svg":
        this.svg_depth += 1
        in_svg = true

    # Handle <link rel="stylesheet"> specially
    if tag == "link":
        try handle_stylesheet_link(this, attributes)
        return

    # Determine variable name
    var var_name = ""
    var is_named = false
    if data_name != "":
        var_name = try luce_identifier(data_name)
        is_named = true
        _ = try this.field_initializers.write(f"    this.{var_name} = gc.make_weak(no_element)\n")
    else:
        this.unnamed_element_counter += 1
        var_name = try cat(["element_", try decimal(this.unnamed_element_counter)])

    let (namespace_expression, tag_expression) = try resolve_tag(this, tag, in_svg)
    let parent = try parent_node(this)

    try add_line_parts(this, ["let ", var_name, " = dom_create_element(document, ", tag_expression, ", ", namespace_expression, ") else trap(\"MUST: DOM::create_element\")"])
    if is_named:
        try add_line_parts(this, ["this.", var_name, " = gc.make_weak(", var_name, ")"])

    # Set attributes (skip data-name)
    for attribute in attributes:
        if attribute.name == "data-option" or attribute.name == "data-name":
            continue
        let attribute_expression = try resolve_attribute(this, attribute.name, in_svg)
        try add_line_parts(this, ["dom_element_set_attribute_value(", var_name, ", ", attribute_expression, ", ak.string_from_str(", try quoted(attribute.value), "))"])

    try add_line_parts(this, ["_ = dom_node_append_child(", parent, ", ", var_name, ".dom_node()) else trap(\"MUST: append_child\")"])

    try add_blank_line(this)

    if this.stack_size == max_depth:
        return error(error_bad_template, "the template nests too deeply")
    this.stack[this.stack_size] = OpenElement(var_name = var_name, tag = tag, option = option)
    this.stack_size += 1

    if self_closing or (is_void_element(tag) and not in_svg):
        try handle_endtag(this, tag)

## Whether `tag` is an HTML void element.
func is_void_element(tag: str) -> bool:
    for name in html_void_elements:
        if name == tag:
            return true
    return false

## `handle_endtag(tag)`.
func handle_endtag(this: Generator*, tag: str) -> !:
    if this.stack_size == 0 or this.stack[this.stack_size - 1].tag != tag:
        return
    this.stack_size -= 1
    let node = this.stack[this.stack_size]

    if tag == "svg":
        this.svg_depth -= 1
    if node.option != "":
        this.indentation -= 1
        if let start = this.blank_line_start:
            this.output.truncate(start)
            this.blank_line_start = none
        let option_case = try pascal_case(node.option)
        try add_line_parts(this, ["# Options::", option_case])
        try add_blank_line(this)

## to_pascal_case: `video` → `Video` (the C++ enumerator's name, for the closing comment).
func pascal_case(text: str) -> str!:
    var builder: collections.Buffer
    var start_of_word = true
    for byte in text.bytes:
        let is_letter_or_digit = (byte >= 'a' and byte <= 'z') or (byte >= 'A' and byte <= 'Z') or (byte >= '0' and byte <= '9')
        if not is_letter_or_digit:
            start_of_word = true
            continue
        var out = byte
        if start_of_word and byte >= 'a' and byte <= 'z':
            out = byte - 32
        elif not start_of_word and byte >= 'A' and byte <= 'Z':
            out = byte + 32
        let one: u8[1] = [out]
        _ = try builder.write(one)
        start_of_word = false
    return builder.text()

## `handle_data(data)`: non-blank text sets the current parent's text content.
func handle_data(this: Generator*, data: str) -> !:
    if strings.trim(data) == "":
        return
    let parent = try parent_node(this)
    try add_line_parts(this, ["dom_node_set_text_content(", parent, ", ak.utf16_string_from_str(", try quoted(data), ")) else trap(\"MUST: set_text_content\")"])
    try add_blank_line(this)

## `_handle_stylesheet_link(attrs)`: a <style> element holding the linked style sheet.
func handle_stylesheet_link(this: Generator*, attributes: const Attribute[]) -> !:
    var href = ""
    for attribute in attributes:
        if attribute.name == "href":
            href = attribute.value
    if href == "":
        return error(error_bad_template, "a <link> without href")
    let css_content = try read_text(try c_cat([this.data_directory, "/", href]))

    this.unnamed_element_counter += 1
    let var_name = try cat(["element_", try decimal(this.unnamed_element_counter)])

    this.stylesheet_count += 1
    let source_var = try cat(["html_media_controls_dom_stylesheet_source_", try decimal(this.stylesheet_count)])
    try add_stylesheet_source(this, source_var, href, css_content)

    let parent = try parent_node(this)

    try add_line_parts(this, ["let ", var_name, " = dom_create_element(document, html_tag_names_style, namespace_html) else trap(\"MUST: DOM::create_element\")"])
    try add_line_parts(this, ["dom_node_set_text_content(", var_name, ".dom_node(), ak.utf16_string_from_str(", source_var, ")) else trap(\"MUST: set_text_content\")"])
    try add_line_parts(this, ["_ = dom_node_append_child(", parent, ", ", var_name, ".dom_node()) else trap(\"MUST: append_child\")"])

    try add_blank_line(this)

## A style sheet source constant: `static String s_stylesheet_source_N = R"~~~(\n<stripped css>\n)~~~"_string;`.
func add_stylesheet_source(this: Generator*, source_var: str, href: str, css_content: str) -> !:
    let stripped = strings.trim(css_content)
    _ = try this.stylesheet_sources.write(f"## The style sheet {href}, stripped (MediaControlsDOM.cpp's `{source_var}`).\n")
    _ = try this.stylesheet_sources.write(f"let {source_var}: str = \"\"\"\n\n")
    for byte in stripped.bytes:
        if byte == '\\':
            try this.stylesheet_sources.put_text("\\\\")
        elif byte == '\t':
            try this.stylesheet_sources.put_text("\\t")
        elif byte == '\r':
            try this.stylesheet_sources.put_text("\\r")
        else:
            let one: u8[1] = [byte]
            _ = try this.stylesheet_sources.write(one)
    try this.stylesheet_sources.put_text("\n\"\"\"\n\n")

# mark: The template scanner ====================================================================

## Whether `byte` is HTML whitespace.
func is_space(byte: u8) -> bool:
    return byte == ' ' or byte == '\n' or byte == '\t' or byte == '\r' or byte == 0x0c

## Tokenize `html` and feed each start tag, end tag and text to the generator (HTMLParser.feed).
func feed(this: Generator*, html: str) -> !:
    let bytes = html.bytes
    var position: usize = 0
    while position < bytes.length:
        if bytes[position] != '<':
            # Text up to the next tag.
            let start = position
            while position < bytes.length and bytes[position] != '<':
                position += 1
            try handle_data(this, (str)bytes[start..<position])
            continue

        # A comment: dropped (handle_comment).
        if strings.starts_with((str)bytes[position..<bytes.length], "<!--"):
            let end = strings.find_from(html, "-->", position + 4) else return error(error_bad_template, "an unterminated comment")
            position = end + 3
            continue

        # An end tag.
        if position + 1 < bytes.length and bytes[position + 1] == '/':
            let name_start = position + 2
            var name_end = name_start
            while name_end < bytes.length and bytes[name_end] != '>' and not is_space(bytes[name_end]):
                name_end += 1
            let close = strings.find_from(html, ">", name_end) else return error(error_bad_template, "an unterminated end tag")
            try handle_endtag(this, (str)bytes[name_start..<name_end])
            position = close + 1
            continue

        # A start tag: its name, then its attributes.
        let name_start = position + 1
        var cursor = name_start
        while cursor < bytes.length and bytes[cursor] != '>' and bytes[cursor] != '/' and not is_space(bytes[cursor]):
            cursor += 1
        let tag = (str)bytes[name_start..<cursor]

        var attributes: Attribute[16] = ---
        var attribute_count: usize = 0
        var self_closing = false
        while true:
            while cursor < bytes.length and is_space(bytes[cursor]):
                cursor += 1
            if cursor >= bytes.length:
                return error(error_bad_template, "an unterminated start tag")
            if bytes[cursor] == '>':
                cursor += 1
                break
            if bytes[cursor] == '/':
                self_closing = true
                cursor += 1
                continue
            let attribute_start = cursor
            while cursor < bytes.length and bytes[cursor] != '=' and bytes[cursor] != '>' and bytes[cursor] != '/' and not is_space(bytes[cursor]):
                cursor += 1
            let attribute_name = (str)bytes[attribute_start..<cursor]
            var attribute_value = ""
            if cursor < bytes.length and bytes[cursor] == '=':
                cursor += 1
                if cursor < bytes.length and (bytes[cursor] == '"' or bytes[cursor] == '\''):
                    let quote = bytes[cursor]
                    let value_start = cursor + 1
                    cursor = value_start
                    while cursor < bytes.length and bytes[cursor] != quote:
                        cursor += 1
                    attribute_value = (str)bytes[value_start..<cursor]
                    cursor += 1
                else:
                    let value_start = cursor
                    while cursor < bytes.length and bytes[cursor] != '>' and not is_space(bytes[cursor]):
                        cursor += 1
                    attribute_value = (str)bytes[value_start..<cursor]
            if attribute_count == 16:
                return error(error_bad_template, "too many attributes")
            attributes[attribute_count] = Attribute(name = attribute_name, value = attribute_value)
            attribute_count += 1
        position = cursor

        try handle_tag(this, tag, attributes[0..<attribute_count], self_closing)

# mark: The fragment ============================================================================

## The generated fragment: header, style sheet sources and the constructor.
func render(this: Generator*) -> str!:
    let rule = "#=============================================================================================="
    var builder: collections.Buffer
    try builder.put_text(rule)
    try builder.put_text("\n#\n")
    try builder.put_text("#   media_controls_dom - HTML::MediaControlsDOM: the DOM of MediaControls.html\n")
    try builder.put_text("#\n#   DESCRIPTION:\n")
    try builder.put_text("#       Generated by tools/gen_dom_tree from data/html/MediaControls.html; do not edit.\n")
    try builder.put_text("#       The C++ it stands for, at Ladybird 47c82b38d0, is\n")
    try builder.put_text("#       Build/Libraries/LibWeb/HTML/MediaControlsDOM.cpp,\n")
    try builder.put_text("#       written by Meta/Generators/generate_dom_tree.py.\n")
    try builder.put_text("#\n#       Copyright (c) 2026, the Ladybird developers.\n")
    try builder.put_text("#       SPDX-License-Identifier: BSD-2-Clause (see data/html/LICENSE)\n#\n")
    try builder.put_text(rule)
    try builder.put_text("\n\n")
    try builder.put(this.stylesheet_sources.view())
    try builder.put_text("## Web::HTML::MediaControlsDOM::MediaControlsDOM(DOM::Document&, DOM::Node& parent, Options): builds the\n")
    try builder.put_text("## template's DOM under `parent`, naming its data-name elements in `this`.\n")
    try builder.put_text("pub func html_media_controls_dom_construct(this: HtmlMediaControlsDom*, document: DomDocument*, parent: DomNode*, options: HtmlMediaControlsDomOptions = HtmlMediaControlsDomOptions.none_):\n")
    try builder.put_text("    # The fields' default member initializers: every GC::Weak starts null.\n")
    try builder.put_text("    let no_element: DomElement*? = none\n")
    try builder.put(this.field_initializers.view())
    try builder.put_text("\n")
    try builder.put(this.output.view())
    return builder.text()

# mark: Text helpers ============================================================================

## The decimal digits of `value`.
func decimal(value: usize) -> str!:
    var buffer: u8[24] = ---
    let text = try strings.format(buffer, f"{value}")
    return try strings.copy(text)

## The concatenation of `parts`.
func cat(parts: const str[]) -> str!:
    var builder: collections.Buffer
    for part in parts:
        try builder.put_text(part)
    return builder.text()

## The concatenation of `parts` as the C string the files module takes.
func c_cat(parts: const str[]) -> c.str!:
    var builder: collections.Buffer
    for part in parts:
        try builder.put_text(part)
    return try builder.terminated()