Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-engine

Source of release 0.6.0

0.6.0 / src / web / dom / document_loading_1.lucb

#==============================================================================================
#
#   document_loading_1 - loading XML, HTML and text documents for a navigation
#
#   DESCRIPTION:
#       Port of LibWeb/DOM/DocumentLoading.cpp (lines 40-302): converting a document into an
#       XML error document, building XML documents, and the HTML, XML and text loaders. The
#       C++ lambdas become capture structs with static `call` functions, wrapped in GC
#       functions as in the donor.
#
#       Copyright (c) 2020, Andreas Kling <andreas@ladybird.org>
#       Copyright (c) 2023, Aliaksandr Kalenik <kalenik.aliaksandr@gmail.com>
#       Copyright (c) 2023, Sam Atkins <atkinssj@serenityos.org>
#       SPDX-License-Identifier: BSD-2-Clause
#
#==============================================================================================

# mark: XML documents ===========================================================================

## Ported from DocumentLoading.cpp:41 Web::convert_to_xml_error_document.
# Replaces a document's content with a simple error message.
func convert_to_xml_error_document(document: DomDocument*, error_string: ak.Utf16String):
    let html_element = dom_create_element(document, html_tag_names_html, namespace_html) else trap("MUST: create_element(html)")
    let body_element = dom_create_element(document, html_tag_names_body, namespace_html) else trap("MUST: create_element(body)")
    _ = dom_node_append_child(html_element.dom_node(), body_element.dom_node()) else trap("MUST: append_child")
    let text = realm_create_dom_text_utf16_string(dom_document_realm(document), document, error_string)
    _ = dom_node_append_child(body_element.dom_node(), text.dom_node()) else trap("MUST: append_child")
    dom_node_remove_all_children(document.dom_node())
    _ = dom_node_append_child(document.dom_node(), html_element.dom_node()) else trap("MUST: append_child")

## The `Optional<TextCodec::Decoder&> decoder` choice shared by build_xml_document and load_xml_document:
## the Content-Type charset when it names a decoder, otherwise the encoding sniffing algorithm's result.
func document_loading_xml_decoder(document: DomDocument*, data: const u8[], content_encoding: ak.String?, mime_type: MimeType?) -> text_codec.Decoder*:
    var decoder: text_codec.Decoder*? = none
    # The actual HTTP headers and other metadata, not the headers as mutated or implied by the algorithms given in this specification,
    # are the ones that must be used when determining the character encoding according to the rules given in the above specifications.
    if let encoding = content_encoding:
        decoder = text_codec.decoder_for(ak.string_bytes_as_string_view(&encoding))
    if decoder == none:
        let encoding = html_run_encoding_sniffing_algorithm(document, data, mime_type)
        decoder = text_codec.decoder_for(ak.byte_string_view(&encoding))
    return decoder else trap("VERIFY: decoder.has_value()")

## Ported from DocumentLoading.cpp:51 Web::build_xml_document.
pub func build_xml_document(document: DomDocument*, data: ak.ByteBuffer, content_encoding: ak.String?) -> bool:
    let bytes = ak.byte_buffer_bytes_const(&data)
    let decoder = document_loading_xml_decoder(document, bytes, content_encoding, none)
    # Well-formed XML documents contain only properly encoded characters
    if not text_codec.decoder_validate(decoder, ak.string_view_from_bytes(bytes)):
        convert_to_xml_error_document(document, ak.utf16_string_from_str("XML Document contains improperly-encoded characters"))
        return false
    let source = text_codec.decoder_to_utf8(decoder, ak.string_view_from_bytes(bytes)) else trap("release_value_but_fixme_should_propagate_errors: Decoder::to_utf8")
    # XML::Parser parser(source, { .resolve_named_html_entity = resolve_named_html_entity });
    # XMLDocumentBuilder builder { document };
    # auto result = parser.parse_with_listener(builder);
    let result = xml_parse_with_document_builder(document, source, XmlParserOptions(resolve_named_html_entity = resolve_named_html_entity))
    return result.parse_error == none and not result.builder_has_error

# mark: HTML documents ==========================================================================

## load_html_document's about:srcdoc deferred parser start: `[parser, url] { parser->run(url); }`.
struct DocumentLoadingRunParserWithUrl:
    var parser: HtmlParser*
    var url: web_url.Url

    static func call(context: void*?):
        let captures = (DocumentLoadingRunParserWithUrl*)(context else trap("run parser: no captures"))
        html_parser_run_url(captures.parser, captures.url)

## Ported from DocumentLoading.cpp:76 Web::load_html_document.
# https://html.spec.whatwg.org/multipage/document-lifecycle.html#navigate-html
func load_html_document(navigation_params: HtmlNavigationParams*) -> DomDocument*!:
    # To load an HTML document, given navigation params navigationParams:
    let response = navigation_params.response else trap("VERIFY: navigation params have a response")
    let body = fetch_response_body(response) else trap("VERIFY: the response has a body")

    # 1. Let document be the result of creating and initializing a Document object given "html", "text/html", and navigationParams.
    let document = try dom_document_create_and_initialize(DocumentDomDocumentType.html, ak.string_from_str("text/html"), navigation_params)

    # 2. If document's URL is about:blank, then populate with html/head/body given document.
    # FIXME: The additional check for a non-empty body fixes issues with loading javascript urls in iframes, which
    #        default to an "about:blank" url. Is this a spec bug?
    let url_string = dom_document_url_string(document)
    let body_length = fetch_body_length(body) else 0
    let srcdoc_data = fetch_body_source_byte_buffer(body)
    let document_url = dom_document_url(document)
    if ak.string_eq_string_view(&url_string, ak.sv("about:blank")) and body_length == 0:
        try dom_document_populate_with_html_head_and_body(document)
        html_parser_the_end(document)

    # AD-HOC: For about:srcdoc, the body bytes are always immediately available in the response source (the srcdoc
    #         string was inlined when the navigation params were created). Bypass the async body-reading pipeline
    #         and set up a deferred parser directly. Combined with running the post-activation update synchronously
    #         when a deferred parser is set, this guarantees the body element exists before the document becomes
    #         observable to the parent — matching Chrome and Firefox behavior for srcdoc iframes.
    #
    # FIXME: This only fixes the transient `contentDocument.body === null` race for srcdoc. The same race exists in
    #        Ladybird for any other same-origin async iframe load (notably blob: URLs and same-origin HTTP), where
    #        Chrome and Firefox also keep `contentDocument` pointed at the initial about:blank until the new document
    #        reaches readyState="interactive". A spec-aligned fix would split "what the parent sees via
    #        contentDocument" from the navigable's active document, swapping the parent-visible pointer only at
    #        parser readiness.
    elif srcdoc_data != none and web_url.url_eq(&document_url, web_url.about_srcdoc()):
        let data = srcdoc_data else trap("unreachable: checked above")
        let mime_type = fetch_extract_mime_type(fetch_response_header_list(response))
        let url = fetch_response_url(response) else trap("VERIFY: the response has a URL")
        let parser = html_parser_create_with_uncertain_encoding(document, ak.byte_buffer_clone(data), mime_type)
        let captures = new DocumentLoadingRunParserWithUrl(parser = parser, url = url) else trap("out of memory")
        dom_document_set_deferred_parser_start(document, gc.create_function0[unit](gc.cell_heap(document.cell()), ak.Function0[unit](call = DocumentLoadingRunParserWithUrl.call, context = captures)))

    # 3. Otherwise, create an HTML parser and associate it with the document.
    #    Each task that the networking task source places on the task queue while fetching runs must then fill the
    #    parser's input byte stream with the fetched bytes and cause the HTML parser to perform the appropriate
    #    processing of the input stream.
    #    The first task that the networking task source places on the task queue while fetching runs must process link
    #    headers given document, navigationParams's response, and "media", after the task has been processed by the
    #    HTML parser.
    #    Before any script execution occurs, the user agent must wait for scripts may run for the newly-created
    #    document to be true for document.
    #    When no more bytes are available, the user agent must queue a global task on the networking task source given
    #    document's relevant global object to have the parser to process the implied EOF character, which eventually
    #    causes a load event to be fired.
    else:
        let url = fetch_response_url(response) else trap("VERIFY: the response has a URL")
        let parser = html_incremental_document_parser_create(document, body, url, fetch_extract_mime_type(fetch_response_header_list(response)))
        html_incremental_document_parser_start(parser)

    # 4. Return document.
    return document

# mark: XML documents (navigation) ==============================================================

## load_xml_document's `run_xml_parser` lambda: `[document, source_string]`.
struct DocumentLoadingRunXmlParser:
    var document: DomDocument*
    var source_string: ak.String

    static func call(context: void*?):
        let captures = (DocumentLoadingRunXmlParser*)(context else trap("run_xml_parser: no captures"))
        let document = captures.document
        # XML::Parser parser(source_string, { .preserve_cdata = true, .preserve_comments = true, .resolve_named_html_entity = resolve_named_html_entity });
        # XMLDocumentBuilder builder { document };
        # auto result = parser.parse_with_listener(builder);
        let result = xml_parse_with_document_builder(document, captures.source_string, XmlParserOptions(preserve_cdata = true, preserve_comments = true, resolve_named_html_entity = resolve_named_html_entity))
        if let parse_error = result.parse_error:
            # FIXME: Insert error message into the document.
            var builder = ak.string_builder_new()
            ak.string_builder_append(&builder, "Failed to parse XML document: ")
            ak.string_builder_append_string_view(&builder, ak.string_bytes_as_string_view(&parse_error))
            ak.dbgln(f"{ak.string_view_text(ak.string_builder_string_view(&builder))}")
            convert_to_xml_error_document(document, ak.utf16_string_from_utf8_string_view(ak.string_builder_string_view(&builder)))

            # NB: XMLDocumentBuilder ensures that the `load` event gets fired. We don't need to do anything else here.

## load_xml_document's `process_body` lambda: `[document, url, content_encoding, mime = type](ByteBuffer data)`.
## Public for web_test's worker, which loads a test's file this way with the file's bytes (there is
## no body stream to read them from before phase 4's Streams).
pub struct DocumentLoadingProcessXmlBody:
    pub var document: DomDocument*
    pub var url: web_url.Url
    pub var content_encoding: ak.String?
    pub var mime: MimeType

    ## The lambda's steps: decode `data`, then run the XML parser (now, or when scripts may run).
    pub static func call(context: void*?, data: ak.ByteBuffer):
        let captures = (DocumentLoadingProcessXmlBody*)(context else trap("process_body: no captures"))
        let document = captures.document
        let bytes = ak.byte_buffer_bytes_const(&data)
        let decoder = document_loading_xml_decoder(document, bytes, captures.content_encoding, captures.mime)
        # Well-formed XML documents contain only properly encoded characters
        if not text_codec.decoder_validate(decoder, ak.string_view_from_bytes(bytes)):
            # FIXME: Insert error message into the document.
            ak.dbgln("XML Document contains improperly-encoded characters")
            convert_to_xml_error_document(document, ak.utf16_string_from_str("XML Document contains improperly-encoded characters"))

            # NB: This ensures that the `load` event gets fired for the frame loading this document.
            dom_document_completely_finish_loading(document)
            return
        let source = text_codec.decoder_to_utf8(decoder, ak.string_view_from_bytes(bytes)) catch failure:
            # FIXME: Insert error message into the document.
            var builder = ak.string_builder_new()
            ak.string_builder_append(&builder, "Failed to decode XML document: ")
            ak.string_builder_append(&builder, failure.message)
            ak.dbgln(f"{ak.string_view_text(ak.string_builder_string_view(&builder))}")
            convert_to_xml_error_document(document, ak.utf16_string_from_utf8_string_view(ak.string_builder_string_view(&builder)))

            # NB: This ensures that the `load` event gets fired for the frame loading this document.
            dom_document_completely_finish_loading(document)
            return
        let run_xml_parser = new DocumentLoadingRunXmlParser(document = document, source_string = source) else trap("out of memory")
        if dom_document_ready_to_run_scripts(document):
            DocumentLoadingRunXmlParser.call(run_xml_parser)
        else:
            dom_document_set_deferred_parser_start(document, gc.create_function0[unit](gc.cell_heap(document.cell()), ak.Function0[unit](call = DocumentLoadingRunXmlParser.call, context = run_xml_parser)))

## The `process_body_error` lambda of the XML and text loaders.
func document_loading_process_body_error(context: void*?, body_error: JsValue):
    ak.dbgln("FIXME: Load html page with an error if read of body failed.")

## Ported from DocumentLoading.cpp:137 Web::load_xml_document.
# https://html.spec.whatwg.org/multipage/document-lifecycle.html#read-xml
func load_xml_document(navigation_params: HtmlNavigationParams*, type_: MimeType) -> DomDocument*!:
    # When faced with displaying an XML file inline, provided navigation params navigationParams and a string type, user agents
    # must follow the requirements defined in XML and Namespaces in XML, XML Media Types, DOM, and other relevant specifications
    # to create and initialize a Document object document, given "xml", type, and navigationParams, and return that Document.
    # They must also create a corresponding XML parser. [XML] [XMLNS] [RFC7303] [DOM]
    #
    # Note: At the time of writing, the XML specification community had not actually yet specified how XML and the DOM interact.
    #
    # The first task that the networking task source places on the task queue while fetching runs must process link headers
    # given document, navigationParams's response, and "media", after the task has been processed by the XML parser.
    #
    # The actual HTTP headers and other metadata, not the headers as mutated or implied by the algorithms given in this
    # specification, are the ones that must be used when determining the character encoding according to the rules given in the
    # above specifications. Once the character encoding is established, the document's character encoding must be set to that
    # character encoding.
    #
    # Before any script execution occurs, the user agent must wait for scripts may run for the newly-created document to be
    # true for the newly-created Document.
    #
    # Once parsing is complete, the user agent must set document's during-loading navigation ID for WebDriver BiDi to null.
    #
    # Note: For HTML documents this is reset when parsing is complete, after firing the load event.
    #
    # Error messages from the parse process (e.g., XML namespace well-formedness errors) may be reported inline by mutating
    # the Document.

    # FIXME: Actually follow the spec! This is just the ad-hoc code we had before, modified somewhat.

    let document = try dom_document_create_and_initialize(DocumentDomDocumentType.xml, mime_type_essence(&type_), navigation_params)

    var content_encoding: ak.String? = none
    if let maybe_encoding = ak.ordered_hash_map_get_k_const(mime_type_parameters(&type_), ak.string_from_str("charset")):
        content_encoding = *maybe_encoding

    let response = navigation_params.response else trap("VERIFY: navigation params have a response")
    let url = fetch_response_url(response) else trap("VERIFY: the response has a URL")
    let captures = new DocumentLoadingProcessXmlBody(document = document, url = url, content_encoding = content_encoding, mime = type_) else trap("out of memory")
    let process_body = gc.create_function1[ak.ByteBuffer, unit](gc.cell_heap(document.cell()), ak.Function1[ak.ByteBuffer, unit](call = DocumentLoadingProcessXmlBody.call, context = captures))

    let process_body_error = gc.create_function1[JsValue, unit](gc.cell_heap(document.cell()), ak.Function1[JsValue, unit](call = document_loading_process_body_error, context = none))

    let realm = dom_document_realm(document)
    let body = fetch_response_body(response) else trap("VERIFY: the response has a body")
    fetch_body_fully_read(body, realm, process_body, process_body_error, .js_object((JsObject*)realm_global_object(realm)))

    return document

# mark: Text documents ==========================================================================

## load_text_document's `run_text_parser` lambda: `[document, data, url, encoding]`.
struct DocumentLoadingRunTextParser:
    var document: DomDocument*
    var data: ak.ByteBuffer
    var url: web_url.Url
    var encoding: ak.ByteString

    static func call(context: void*?):
        let captures = (DocumentLoadingRunTextParser*)(context else trap("run_text_parser: no captures"))
        let document = captures.document
        let parser = html_parser_create_with_open_input_stream(document)
        let tokenizer = html_parser_tokenizer(parser)
        html_syntax.html_tokenizer_update_insertion_point(tokenizer)

        html_syntax.html_tokenizer_insert_input_at_insertion_point(tokenizer, ak.sv("<pre>\n"))
        html_parser_run_html_tokenizer_stop_at_inserti(parser)

        html_syntax.html_tokenizer_switch_to_html_parser(tokenizer, html_syntax.HtmlTokenizerState.plaintext)
        html_syntax.html_tokenizer_insert_input_at_insertion_point(tokenizer, ak.string_view_from_bytes(ak.byte_buffer_bytes_const(&captures.data)))
        html_syntax.html_tokenizer_insert_eof(tokenizer)
        html_parser_run_url(parser, captures.url)

        dom_document_set_encoding(document, ak.string_from_byte_string(captures.encoding) else trap("MUST: String::from_byte_string"))

        # 5. User agents may add content to the head element of document, e.g., linking to a style sheet, providing
        #    script, or giving the document a title.
        document_loading_insert_title(document, captures.url) else trap("MUST: insert the title")

## `dbgln_if(HTML_PARSER_DEBUG, ...)` (AK/Debug.h; off by default).
let document_loading_html_parser_debug: bool = false

## load_text_document's `process_body` lambda: `[document, url, mime = type](ByteBuffer data)`.
struct DocumentLoadingProcessTextBody:
    var document: DomDocument*
    var url: web_url.Url
    var mime: MimeType

    static func call(context: void*?, data: ak.ByteBuffer):
        let captures = (DocumentLoadingProcessTextBody*)(context else trap("process_body: no captures"))
        let document = captures.document
        let encoding = html_run_encoding_sniffing_algorithm(document, ak.byte_buffer_bytes_const(&data), captures.mime)
        if document_loading_html_parser_debug:
            ak.dbgln(f"The encoding sniffing algorithm returned encoding '{ak.string_view_text(ak.byte_string_view(&encoding))}'")

        let run_text_parser = new DocumentLoadingRunTextParser(document = document, data = data, url = captures.url, encoding = encoding) else trap("out of memory")
        if dom_document_ready_to_run_scripts(document):
            DocumentLoadingRunTextParser.call(run_text_parser)
        else:
            dom_document_set_deferred_parser_start(document, gc.create_function0[unit](gc.cell_heap(document.cell()), ak.Function0[unit](call = DocumentLoadingRunTextParser.call, context = run_text_parser)))

## Ported from DocumentLoading.cpp:233 Web::load_text_document.
# https://html.spec.whatwg.org/multipage/document-lifecycle.html#navigate-text
func load_text_document(navigation_params: HtmlNavigationParams*, type_: MimeType) -> DomDocument*!:
    # To load a text document, given a navigation params navigationParams and a string type:

    # 1. Let document be the result of creating and initializing a Document object given "html", type, and navigationParams.
    let document = try dom_document_create_and_initialize(DocumentDomDocumentType.html, mime_type_essence(&type_), navigation_params)

    # 2. Set document's parser cannot change the mode flag to true.
    dom_document_set_parser_cannot_change_the_mode(document, true)

    # 3. Set document's mode to "no-quirks".
    dom_document_set_quirks_mode(document, DomQuirksMode.no)

    # 4. Create an HTML parser and associate it with the document. Act as if the tokenizer had emitted a start tag token
    #    with the tag name "pre" followed by a single U+000A LINE FEED (LF) character, and switch the HTML parser's tokenizer
    #    to the PLAINTEXT state. Each task that the networking task source places on the task queue while fetching runs must
    #    then fill the parser's input byte stream with the fetched bytes and cause the HTML parser to perform the appropriate
    #    processing of the input stream.
    #    document's encoding must be set to the character encoding used to decode the document during parsing.
    #    The first task that the networking task source places on the task queue while fetching runs must process link
    #    headers given document, navigationParams's response, and "media", after the task has been processed by the HTML parser.
    #    Before any script execution occurs, the user agent must wait for scripts may run for the newly-created document to be
    #    true for document.
    #    When no more bytes are available, the user agent must queue a global task on the networking task source given
    #    document's relevant global object to have the parser to process the implied EOF character, which eventually causes a
    #    load event to be fired.
    # FIXME: Parse as we receive the document data, instead of waiting for the whole document to be fetched first.
    let response = navigation_params.response else trap("VERIFY: navigation params have a response")
    let url = fetch_response_url(response) else trap("VERIFY: the response has a URL")
    let captures = new DocumentLoadingProcessTextBody(document = document, url = url, mime = type_) else trap("out of memory")
    let process_body = gc.create_function1[ak.ByteBuffer, unit](gc.cell_heap(document.cell()), ak.Function1[ak.ByteBuffer, unit](call = DocumentLoadingProcessTextBody.call, context = captures))

    let process_body_error = gc.create_function1[JsValue, unit](gc.cell_heap(document.cell()), ak.Function1[JsValue, unit](call = document_loading_process_body_error, context = none))

    let realm = dom_document_realm(document)
    let body = fetch_response_body(response) else trap("VERIFY: the response has a body")
    fetch_body_fully_read(body, realm, process_body, process_body_error, .js_object((JsObject*)realm_global_object(realm)))

    # 6. Return document.
    return document