Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-engine

Source of release 0.6.0

0.6.0 / src / web / html / parser / tests_html_parser_2.lucb

#==============================================================================================
#
#   tests_html_parser_2 - tests of HTMLParser's algorithms on a hand-built parser (region r21)
#
#   DESCRIPTION:
#       The parts of the tree builder that do not create or insert nodes, run on a parser made
#       by hand (tests_html_parser_support) over a stack of hand-made elements: quirks mode
#       from a DOCTYPE (the reference build's values, from a local oracle compiling the donor's
#       which_quirks_mode), the special category and the integration points, the tree
#       construction dispatcher, implied end tags, clearing the stack back to a table context,
#       resetting the insertion mode (document and fragment cases), the insertion-mode entries
#       that only pop elements and switch modes, the adoption agency algorithm's early returns,
#       the appropriate place for inserting a node under foster parenting, the foreign-content
#       token adjustments, escaping and the HTML fragment serialization of a hand-built tree,
#       and parse_dimension_value's failures.
#
#       What needs regions r17 (Document), r22/r26/r27 (HTML element constructors), r29 (the
#       relevant agent) and r42 (the custom-element state invalidation every created element
#       runs) waits for them: docs/regions/r21.md lists the scenario tests to add then.
#
#==============================================================================================

# mark: Helpers =================================================================================

## A hand-built parser over `world`'s document whose stack holds HTML elements named `names`.
func parser_test_parser_with_stack(world: ParserTestWorld*, names: const str[], mode: HtmlParserInsertionMode) -> HtmlParser*:
    let parser = parser_test_html_parser(world.realm, world.document, "")
    parser_test_push_html(world, &parser.m_stack_of_open_elements, names)
    parser.m_insertion_mode = mode
    return parser

## Processes a start or end tag named `name` in the parser's insertion mode.
func parser_test_process(parser: HtmlParser*, type_: html_syntax.HtmlTokenType, name: str):
    var token = parser_test_tag(type_, name)
    html_parser_process_using_the_rules_for(parser, parser.m_insertion_mode, &token)

## A row of the quirks-mode table.
struct QuirksCase:
    var name: str
    var public_identifier: str
    var system_identifier: str
    var missing_public_identifier: bool
    var missing_system_identifier: bool
    var force_quirks: bool
    var expected: DomQuirksMode

# mark: Quirks mode and categories ==============================================================

test "HTMLParser::which_quirks_mode matches the reference build":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)
    let parser = parser_test_html_parser(world.realm, world.document, "")

    let cases: QuirksCase[20] = [
        QuirksCase(name = "html", public_identifier = "", system_identifier = "", missing_public_identifier = true, missing_system_identifier = true, force_quirks = false, expected = DomQuirksMode.no),
        QuirksCase(name = "html", public_identifier = "", system_identifier = "", missing_public_identifier = true, missing_system_identifier = true, force_quirks = true, expected = DomQuirksMode.yes),
        QuirksCase(name = "HTML", public_identifier = "", system_identifier = "", missing_public_identifier = true, missing_system_identifier = true, force_quirks = false, expected = DomQuirksMode.yes),
        QuirksCase(name = "svg", public_identifier = "", system_identifier = "", missing_public_identifier = true, missing_system_identifier = true, force_quirks = false, expected = DomQuirksMode.yes),
        QuirksCase(name = "html", public_identifier = "-//W3O//DTD W3 HTML Strict 3.0//EN//", system_identifier = "", missing_public_identifier = false, missing_system_identifier = true, force_quirks = false, expected = DomQuirksMode.yes),
        QuirksCase(name = "html", public_identifier = "-/W3C/DTD HTML 4.0 Transitional/EN", system_identifier = "", missing_public_identifier = false, missing_system_identifier = true, force_quirks = false, expected = DomQuirksMode.yes),
        QuirksCase(name = "html", public_identifier = "html", system_identifier = "", missing_public_identifier = false, missing_system_identifier = true, force_quirks = false, expected = DomQuirksMode.yes),
        QuirksCase(name = "html", public_identifier = "", system_identifier = "http://www.ibm.com/data/dtd/v11/IBMXHTML1-transitional.dtd", missing_public_identifier = true, missing_system_identifier = false, force_quirks = false, expected = DomQuirksMode.yes),
        QuirksCase(name = "html", public_identifier = "-//IETF//DTD HTML 2.0//EN", system_identifier = "", missing_public_identifier = false, missing_system_identifier = true, force_quirks = false, expected = DomQuirksMode.yes),
        QuirksCase(name = "html", public_identifier = "-//w3c//dtd html 3.2 final//en", system_identifier = "", missing_public_identifier = false, missing_system_identifier = true, force_quirks = false, expected = DomQuirksMode.yes),
        QuirksCase(name = "html", public_identifier = "-//W3C//DTD HTML 4.01 Frameset//EN", system_identifier = "", missing_public_identifier = false, missing_system_identifier = true, force_quirks = false, expected = DomQuirksMode.yes),
        QuirksCase(name = "html", public_identifier = "-//W3C//DTD HTML 4.01 Frameset//EN", system_identifier = "http://www.w3.org/TR/html4/frameset.dtd", missing_public_identifier = false, missing_system_identifier = false, force_quirks = false, expected = DomQuirksMode.limited),
        QuirksCase(name = "html", public_identifier = "-//W3C//DTD HTML 4.01 Transitional//EN", system_identifier = "", missing_public_identifier = false, missing_system_identifier = true, force_quirks = false, expected = DomQuirksMode.yes),
        QuirksCase(name = "html", public_identifier = "-//W3C//DTD HTML 4.01 Transitional//EN", system_identifier = "http://www.w3.org/TR/html4/loose.dtd", missing_public_identifier = false, missing_system_identifier = false, force_quirks = false, expected = DomQuirksMode.limited),
        QuirksCase(name = "html", public_identifier = "-//W3C//DTD XHTML 1.0 Transitional//EN", system_identifier = "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd", missing_public_identifier = false, missing_system_identifier = false, force_quirks = false, expected = DomQuirksMode.limited),
        QuirksCase(name = "html", public_identifier = "-//W3C//DTD XHTML 1.0 Frameset//EN", system_identifier = "", missing_public_identifier = false, missing_system_identifier = true, force_quirks = false, expected = DomQuirksMode.limited),
        QuirksCase(name = "html", public_identifier = "-//W3C//DTD HTML 4.01//EN", system_identifier = "http://www.w3.org/TR/html4/strict.dtd", missing_public_identifier = false, missing_system_identifier = false, force_quirks = false, expected = DomQuirksMode.no),
        QuirksCase(name = "html", public_identifier = "", system_identifier = "about:legacy-compat", missing_public_identifier = true, missing_system_identifier = false, force_quirks = false, expected = DomQuirksMode.no),
        QuirksCase(name = "html", public_identifier = "-//WebTechs//DTD Mozilla HTML//", system_identifier = "", missing_public_identifier = false, missing_system_identifier = true, force_quirks = false, expected = DomQuirksMode.yes),
        QuirksCase(name = "html", public_identifier = "-//WebTechs//DTD Mozilla HTML", system_identifier = "x", missing_public_identifier = false, missing_system_identifier = false, force_quirks = false, expected = DomQuirksMode.no),
    ]
    for case in cases:
        var token: html_syntax.HtmlToken
        html_syntax.html_token_construct(&token, html_syntax.HtmlTokenType.doctype)
        let data = html_syntax.html_token_ensure_doctype_data(&token)
        data.name = ak.string_from_str(case.name)
        data.missing_name = false
        data.public_identifier = ak.string_from_str(case.public_identifier)
        data.system_identifier = ak.string_from_str(case.system_identifier)
        data.missing_public_identifier = case.missing_public_identifier
        data.missing_system_identifier = case.missing_system_identifier
        data.force_quirks = case.force_quirks
        if html_parser_which_quirks_mode(parser, &token) != case.expected:
            print(f"which_quirks_mode: {case.name}|{case.public_identifier}|{case.system_identifier}")
            assert(false, "quirks mode mismatch")

test "HTMLParser::is_special_tag and the integration points":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)

    let specials: str[8] = ["address", "body", "html", "li", "p", "template", "xmp", "wbr"]
    for name in specials:
        assert(html_parser_is_special_tag(ak.fly_string_from_str(name), namespace_html))
    let not_specials: str[6] = ["a", "b", "span", "font", "nobr", "option"]
    for name in not_specials:
        assert(not html_parser_is_special_tag(ak.fly_string_from_str(name), namespace_html))
    assert(html_parser_is_special_tag(ak.fly_string_from_str("foreignObject"), namespace_svg))
    assert(not html_parser_is_special_tag(ak.fly_string_from_str("foreignobject"), namespace_svg))
    assert(html_parser_is_special_tag(ak.fly_string_from_str("annotation-xml"), namespace_math_ml))
    assert(not html_parser_is_special_tag(ak.fly_string_from_str("p"), namespace_svg))
    assert(not html_parser_is_special_tag(ak.fly_string_from_str("p"), none))

    assert(html_is_mathml_text_integration_point(parser_test_foreign(&world, "mtext", namespace_math_ml)))
    assert(not html_is_mathml_text_integration_point(parser_test_foreign(&world, "math", namespace_math_ml)))
    assert(html_is_html_integration_point(parser_test_foreign(&world, "desc", namespace_svg)))
    assert(not html_is_html_integration_point(parser_test_foreign(&world, "desc", namespace_math_ml)))
    # annotation-xml is an HTML integration point only with encoding="text/html" or "application/xhtml+xml".
    assert(not html_is_html_integration_point(parser_test_foreign(&world, "annotation-xml", namespace_math_ml)))

test "HTMLParser: the tree construction dispatcher":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)
    let parser = parser_test_html_parser(world.realm, world.document, "")
    var start_tag = parser_test_tag(html_syntax.HtmlTokenType.start_tag, "b")
    var character = html_syntax.html_token_make_character((u32)'x')
    var eof: html_syntax.HtmlToken
    html_syntax.html_token_construct(&eof, html_syntax.HtmlTokenType.end_of_file)

    # An empty stack, an HTML current node, an end-of-file token: HTML content.
    assert(html_parser_dispatches_to_html_content(parser, &start_tag))
    parser_test_push_html(&world, &parser.m_stack_of_open_elements, ["html"])
    assert(html_parser_dispatches_to_html_content(parser, &start_tag))
    html_stack_of_open_elements_push(&parser.m_stack_of_open_elements, parser_test_foreign(&world, "svg", namespace_svg))
    assert(not html_parser_dispatches_to_html_content(parser, &start_tag))
    assert(not html_parser_dispatches_to_html_content(parser, &character))
    assert(html_parser_dispatches_to_html_content(parser, &eof))
    # An HTML integration point takes start tags and characters as HTML content.
    html_stack_of_open_elements_push(&parser.m_stack_of_open_elements, parser_test_foreign(&world, "title", namespace_svg))
    assert(html_parser_dispatches_to_html_content(parser, &start_tag))
    assert(html_parser_dispatches_to_html_content(parser, &character))
    # A MathML text integration point, except for mglyph and malignmark.
    html_stack_of_open_elements_push(&parser.m_stack_of_open_elements, parser_test_foreign(&world, "mi", namespace_math_ml))
    assert(html_parser_dispatches_to_html_content(parser, &start_tag))
    var mglyph = parser_test_tag(html_syntax.HtmlTokenType.start_tag, "mglyph")
    assert(not html_parser_dispatches_to_html_content(parser, &mglyph))
    # annotation-xml takes an svg start tag as HTML content.
    html_stack_of_open_elements_push(&parser.m_stack_of_open_elements, parser_test_foreign(&world, "annotation-xml", namespace_math_ml))
    var svg = parser_test_tag(html_syntax.HtmlTokenType.start_tag, "svg")
    assert(html_parser_dispatches_to_html_content(parser, &svg))
    assert(not html_parser_dispatches_to_html_content(parser, &start_tag))
    # The tokenizer's question: is the adjusted current node foreign?
    assert(html_parser_adjusted_current_node_is_foreign(parser))
    let empty_parser = parser_test_html_parser(world.realm, world.document, "")
    assert(not html_parser_adjusted_current_node_is_foreign(empty_parser))

# mark: Implied end tags and the insertion mode =================================================

test "HTMLParser: generating implied end tags and closing a p element":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)

    let parser = parser_test_parser_with_stack(&world, ["html", "body", "p", "li", "dd", "rtc"], HtmlParserInsertionMode.in_body)
    html_parser_generate_implied_end_tags(parser, ak.fly_string_from_str("li"))
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body p li")
    html_parser_close_a_p_element(parser)
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body")

    let thorough = parser_test_parser_with_stack(&world, ["html", "body", "table", "tbody", "tr", "td", "p"], HtmlParserInsertionMode.in_body)
    html_parser_generate_all_implied_end_tags_thoroughly(thorough)
    parser_test_expect_stack(&thorough.m_stack_of_open_elements, "html body table")
    html_parser_generate_implied_end_tags(thorough)
    parser_test_expect_stack(&thorough.m_stack_of_open_elements, "html body table")

test "HTMLParser: clearing the stack back to a table, body and row context":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)

    let parser = parser_test_parser_with_stack(&world, ["html", "body", "table", "tbody", "tr", "td", "b"], HtmlParserInsertionMode.in_body)
    html_parser_clear_the_stack_back_to_a_table_row_context(parser)
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body table tbody tr")
    html_parser_clear_the_stack_back_to_a_table_body_context(parser)
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body table tbody")
    html_parser_clear_the_stack_back_to_a_table_context(parser)
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body table")

test "HTMLParser: resetting the insertion mode appropriately":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)

    let names: (str, HtmlParserInsertionMode)[9] = [
        ("td", HtmlParserInsertionMode.in_cell),
        ("th", HtmlParserInsertionMode.in_cell),
        ("tr", HtmlParserInsertionMode.in_row),
        ("thead", HtmlParserInsertionMode.in_table_body),
        ("caption", HtmlParserInsertionMode.in_caption),
        ("colgroup", HtmlParserInsertionMode.in_column_group),
        ("table", HtmlParserInsertionMode.in_table),
        ("head", HtmlParserInsertionMode.in_head),
        ("body", HtmlParserInsertionMode.in_body),
    ]
    for row in names:
        let parser = parser_test_parser_with_stack(&world, ["html", "div", row.0], HtmlParserInsertionMode.initial)
        html_parser_reset_the_insertion_mode_appropriately(parser)
        assert(parser.m_insertion_mode == row.1)

    # Foreign elements are skipped; the html element answers "after head" once there is a head.
    let parser = parser_test_parser_with_stack(&world, ["html"], HtmlParserInsertionMode.initial)
    html_stack_of_open_elements_push(&parser.m_stack_of_open_elements, parser_test_foreign(&world, "td", namespace_svg))
    parser.m_head_element = (HtmlHeadElement*?)parser_test_html(&world, "head")
    html_parser_reset_the_insertion_mode_appropriately(parser)
    assert(parser.m_insertion_mode == HtmlParserInsertionMode.after_head)

    # A template answers the current template insertion mode.
    let template_parser = parser_test_parser_with_stack(&world, ["html", "template"], HtmlParserInsertionMode.initial)
    ak.vector_append_t(&template_parser.m_stack_of_template_insertion_modes, HtmlParserInsertionMode.in_table_body)
    html_parser_reset_the_insertion_mode_appropriately(template_parser)
    assert(template_parser.m_insertion_mode == HtmlParserInsertionMode.in_table_body)

    # Fragment case: the context element stands for the first node; a td there is not "in cell".
    let contexts: (str, HtmlParserInsertionMode)[5] = [
        ("td", HtmlParserInsertionMode.in_body),
        ("tr", HtmlParserInsertionMode.in_row),
        ("frameset", HtmlParserInsertionMode.in_frameset),
        ("html", HtmlParserInsertionMode.before_head),
        ("select", HtmlParserInsertionMode.in_body),
    ]
    for row in contexts:
        let fragment_parser = parser_test_parser_with_stack(&world, ["html"], HtmlParserInsertionMode.initial)
        fragment_parser.m_parsing_fragment = true
        fragment_parser.m_context_element = parser_test_html(&world, row.0)
        html_parser_reset_the_insertion_mode_appropriately(fragment_parser)
        assert(fragment_parser.m_insertion_mode == row.1)
        assert(html_parser_adjusted_current_node(fragment_parser) == fragment_parser.m_context_element)

test "HTMLParser: insertion_mode_name":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)
    let parser = parser_test_html_parser(world.realm, world.document, "")
    assert(html_parser_insertion_mode_name(parser) == "Initial")
    parser.m_insertion_mode = HtmlParserInsertionMode.before_html
    assert(html_parser_insertion_mode_name(parser) == "BeforeHTML")
    parser.m_insertion_mode = HtmlParserInsertionMode.after_after_frameset
    assert(html_parser_insertion_mode_name(parser) == "AfterAfterFrameset")

# mark: Insertion modes that pop ================================================================

test "HTMLParser: table end tags pop to their context and switch modes":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)
    let end_tag = html_syntax.HtmlTokenType.end_tag

    let parser = parser_test_parser_with_stack(&world, ["html", "body", "table", "tbody", "tr", "td", "b"], HtmlParserInsertionMode.in_cell)
    html_list_of_active_formatting_elements_add_marker(&parser.m_list_of_active_formatting_elements)
    var b_token = parser_test_tag(html_syntax.HtmlTokenType.start_tag, "b")
    html_list_of_active_formatting_elements_add(&parser.m_list_of_active_formatting_elements, html_stack_of_open_elements_current_node(&parser.m_stack_of_open_elements), &b_token)

    # </td> in "in cell": implied end tags, pop to the td, clear the list to the marker, "in row".
    parser_test_process(parser, end_tag, "td")
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body table tbody tr")
    assert(html_list_of_active_formatting_elements_is_empty(&parser.m_list_of_active_formatting_elements))
    assert(parser.m_insertion_mode == HtmlParserInsertionMode.in_row)

    # A stray </td> in "in row" is ignored.
    parser_test_process(parser, end_tag, "td")
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body table tbody tr")

    # </tr> in "in row", </tbody> in "in table body", </table> in "in table".
    parser_test_process(parser, end_tag, "tr")
    assert(parser.m_insertion_mode == HtmlParserInsertionMode.in_table_body)
    parser_test_process(parser, end_tag, "tbody")
    assert(parser.m_insertion_mode == HtmlParserInsertionMode.in_table)
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body table")
    parser_test_process(parser, end_tag, "table")
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body")
    assert(parser.m_insertion_mode == HtmlParserInsertionMode.in_body)

test "HTMLParser: </table> in a row closes the row, the body and the table":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)

    let parser = parser_test_parser_with_stack(&world, ["html", "body", "table", "tbody", "tr"], HtmlParserInsertionMode.in_row)
    parser_test_process(parser, html_syntax.HtmlTokenType.end_tag, "table")
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body")
    assert(parser.m_insertion_mode == HtmlParserInsertionMode.in_body)

test "HTMLParser: caption and column group end tags":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)
    let end_tag = html_syntax.HtmlTokenType.end_tag

    let parser = parser_test_parser_with_stack(&world, ["html", "body", "table", "caption", "p"], HtmlParserInsertionMode.in_caption)
    html_list_of_active_formatting_elements_add_marker(&parser.m_list_of_active_formatting_elements)
    parser_test_process(parser, end_tag, "caption")
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body table")
    assert(parser.m_insertion_mode == HtmlParserInsertionMode.in_table)
    assert(html_list_of_active_formatting_elements_is_empty(&parser.m_list_of_active_formatting_elements))

    html_stack_of_open_elements_push(&parser.m_stack_of_open_elements, parser_test_html(&world, "colgroup"))
    parser.m_insertion_mode = HtmlParserInsertionMode.in_column_group
    parser_test_process(parser, end_tag, "col")
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body table colgroup")
    parser_test_process(parser, end_tag, "colgroup")
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body table")
    assert(parser.m_insertion_mode == HtmlParserInsertionMode.in_table)

test "HTMLParser: head, noscript, body and html end tags switch modes":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)
    let end_tag = html_syntax.HtmlTokenType.end_tag

    let parser = parser_test_parser_with_stack(&world, ["html", "head", "noscript"], HtmlParserInsertionMode.in_head_noscript)
    parser_test_process(parser, end_tag, "noscript")
    assert(parser.m_insertion_mode == HtmlParserInsertionMode.in_head)
    parser_test_process(parser, end_tag, "head")
    assert(parser.m_insertion_mode == HtmlParserInsertionMode.after_head)
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html")

    let body = parser_test_parser_with_stack(&world, ["html", "body", "p"], HtmlParserInsertionMode.in_body)
    parser_test_process(body, end_tag, "body")
    assert(body.m_insertion_mode == HtmlParserInsertionMode.after_body)
    parser_test_process(body, end_tag, "html")
    assert(body.m_insertion_mode == HtmlParserInsertionMode.after_after_body)

    # </html> in "in body" switches to "after body" and reprocesses the token there.
    let html = parser_test_parser_with_stack(&world, ["html", "body"], HtmlParserInsertionMode.in_body)
    parser_test_process(html, end_tag, "html")
    assert(html.m_insertion_mode == HtmlParserInsertionMode.after_after_body)

    # In the "text" insertion mode, an end tag pops and returns to the original mode.
    let text = parser_test_parser_with_stack(&world, ["html", "body", "textarea"], HtmlParserInsertionMode.text)
    text.m_original_insertion_mode = HtmlParserInsertionMode.in_body
    parser_test_process(text, end_tag, "textarea")
    assert(text.m_insertion_mode == HtmlParserInsertionMode.in_body)
    parser_test_expect_stack(&text.m_stack_of_open_elements, "html body")

test "HTMLParser: block, list and heading end tags in body":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)
    let end_tag = html_syntax.HtmlTokenType.end_tag

    let parser = parser_test_parser_with_stack(&world, ["html", "body", "div", "ul", "li", "p"], HtmlParserInsertionMode.in_body)
    parser_test_process(parser, end_tag, "li")
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body div ul")
    parser_test_process(parser, end_tag, "div")
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body")
    # An end tag for an element that is not in scope is ignored.
    parser_test_process(parser, end_tag, "section")
    parser_test_expect_stack(&parser.m_stack_of_open_elements, "html body")

    let headings = parser_test_parser_with_stack(&world, ["html", "body", "h1", "span"], HtmlParserInsertionMode.in_body)
    parser_test_process(headings, end_tag, "h3")
    parser_test_expect_stack(&headings.m_stack_of_open_elements, "html body")

    # Any other end tag pops up to the matching element unless a special element comes first.
    let other = parser_test_parser_with_stack(&world, ["html", "body", "span", "em"], HtmlParserInsertionMode.in_body)
    parser_test_process(other, end_tag, "span")
    parser_test_expect_stack(&other.m_stack_of_open_elements, "html body")
    let blocked = parser_test_parser_with_stack(&world, ["html", "body", "span", "div"], HtmlParserInsertionMode.in_body)
    parser_test_process(blocked, end_tag, "span")
    parser_test_expect_stack(&blocked.m_stack_of_open_elements, "html body span div")
    parser_test_process(blocked, end_tag, "sarcasm")
    parser_test_expect_stack(&blocked.m_stack_of_open_elements, "html body span div")

test "HTMLParser: the adoption agency algorithm's early returns":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)
    var b_token = parser_test_tag(html_syntax.HtmlTokenType.start_tag, "b")
    var b_end = parser_test_tag(html_syntax.HtmlTokenType.end_tag, "b")

    # The current node is a b that is not in the list: pop it.
    let simple = parser_test_parser_with_stack(&world, ["html", "body", "b"], HtmlParserInsertionMode.in_body)
    assert(html_parser_run_the_adoption_agency_algorithm(simple, &b_end) == HtmlParserAdoptionAgencyAlgorithmOutcome.do_nothing)
    parser_test_expect_stack(&simple.m_stack_of_open_elements, "html body")

    # No b in the list (and not the current node): any other end tag steps.
    let none_listed = parser_test_parser_with_stack(&world, ["html", "body", "b", "i"], HtmlParserInsertionMode.in_body)
    assert(html_parser_run_the_adoption_agency_algorithm(none_listed, &b_end) == HtmlParserAdoptionAgencyAlgorithmOutcome.run_any_other_end_tag_steps)

    # A listed b that is not on the stack is removed from the list.
    let off_stack = parser_test_parser_with_stack(&world, ["html", "body", "i"], HtmlParserInsertionMode.in_body)
    html_list_of_active_formatting_elements_add(&off_stack.m_list_of_active_formatting_elements, parser_test_html(&world, "b"), &b_token)
    assert(html_parser_run_the_adoption_agency_algorithm(off_stack, &b_end) == HtmlParserAdoptionAgencyAlgorithmOutcome.do_nothing)
    assert(html_list_of_active_formatting_elements_is_empty(&off_stack.m_list_of_active_formatting_elements))

    # A listed b that is on the stack but not in scope is left alone.
    let out_of_scope = parser_test_parser_with_stack(&world, ["html", "body"], HtmlParserInsertionMode.in_body)
    let b = parser_test_html(&world, "b")
    html_stack_of_open_elements_push(&out_of_scope.m_stack_of_open_elements, b)
    html_list_of_active_formatting_elements_add(&out_of_scope.m_list_of_active_formatting_elements, b, &b_token)
    html_stack_of_open_elements_push(&out_of_scope.m_stack_of_open_elements, parser_test_html(&world, "table"))
    assert(html_parser_run_the_adoption_agency_algorithm(out_of_scope, &b_end) == HtmlParserAdoptionAgencyAlgorithmOutcome.do_nothing)
    parser_test_expect_stack(&out_of_scope.m_stack_of_open_elements, "html body b table")

    # No furthest block: pop up to and including b, and remove it from the list.
    let no_block = parser_test_parser_with_stack(&world, ["html", "body"], HtmlParserInsertionMode.in_body)
    let b2 = parser_test_html(&world, "b")
    html_stack_of_open_elements_push(&no_block.m_stack_of_open_elements, b2)
    html_list_of_active_formatting_elements_add(&no_block.m_list_of_active_formatting_elements, b2, &b_token)
    html_stack_of_open_elements_push(&no_block.m_stack_of_open_elements, parser_test_html(&world, "i"))
    html_stack_of_open_elements_push(&no_block.m_stack_of_open_elements, parser_test_html(&world, "span"))
    parser_test_process(no_block, html_syntax.HtmlTokenType.end_tag, "b")
    parser_test_expect_stack(&no_block.m_stack_of_open_elements, "html body")
    assert(html_list_of_active_formatting_elements_is_empty(&no_block.m_list_of_active_formatting_elements))

test "HTMLParser: end of file in body stops parsing":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)
    let parser = parser_test_parser_with_stack(&world, ["html", "body", "div"], HtmlParserInsertionMode.in_body)
    var eof: html_syntax.HtmlToken
    html_syntax.html_token_construct(&eof, html_syntax.HtmlTokenType.end_of_file)
    html_parser_process_using_the_rules_for(parser, parser.m_insertion_mode, &eof)
    assert(html_parser_stopped(parser))

    # In "after body", a stray end tag is a parse error that returns to "in body".
    let after_body = parser_test_parser_with_stack(&world, ["html", "body"], HtmlParserInsertionMode.after_body)
    parser_test_process(after_body, html_syntax.HtmlTokenType.end_tag, "span")
    assert(after_body.m_insertion_mode == HtmlParserInsertionMode.in_body)

# mark: Insertion locations =====================================================================

test "HTMLParser: the appropriate place for inserting a node, with foster parenting":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)

    let parser = parser_test_parser_with_stack(&world, ["html", "body"], HtmlParserInsertionMode.in_table)
    let body = html_stack_of_open_elements_current_node(&parser.m_stack_of_open_elements)
    let table = parser_test_html(&world, "table")
    html_stack_of_open_elements_push(&parser.m_stack_of_open_elements, table)

    # Without foster parenting: inside the current node, after its last child.
    var location = html_parser_find_appropriate_place_for_inserting_node(parser)
    assert(location.parent == table.dom_node() and location.insert_before_sibling == none)

    # With foster parenting and a table without a parent: inside the element above the table.
    parser.m_foster_parenting = true
    location = html_parser_find_appropriate_place_for_inserting_node(parser)
    assert(location.parent == body.dom_node() and location.insert_before_sibling == none)

    # With a parent: before the table, in its parent.
    let wrapper = parser_test_html(&world, "div")
    test_link_child(wrapper.dom_node(), table.dom_node())
    location = html_parser_find_appropriate_place_for_inserting_node(parser)
    assert(location.parent == wrapper.dom_node() and location.insert_before_sibling == table.dom_node())

    # An override target that is not a table element is used as is.
    location = html_parser_find_appropriate_place_for_inserting_node(parser, body)
    assert(location.parent == body.dom_node())

# mark: Token adjustments =======================================================================

test "HTMLParser: adjusting SVG and MathML names and foreign attributes":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)
    let parser = parser_test_html_parser(world.realm, world.document, "")

    var token = parser_test_tag(html_syntax.HtmlTokenType.start_tag, "foreignobject")
    let names: str[4] = ["viewbox", "definitionurl", "xlink:href", "xmlns"]
    for name in names:
        var attribute = html_syntax.html_token_attribute_new()
        attribute.local_name = ak.fly_string_from_str(name)
        attribute.value = ak.string_from_str("v")
        html_syntax.html_token_add_attribute(&token, attribute)

    html_parser_adjust_svg_tag_names(parser, &token)
    assert(test_fly_is(html_syntax.html_token_tag_name(&token), "foreignObject"))
    html_parser_adjust_svg_attributes(parser, &token)
    html_parser_adjust_mathml_attributes(parser, &token)
    html_parser_adjust_foreign_attributes(&token)

    let attributes = html_syntax.html_token_tag_attributes(&token) else trap("attributes")
    assert(test_fly_is(ak.vector_at(attributes, 0).local_name, "viewBox"))
    assert(test_fly_is(ak.vector_at(attributes, 1).local_name, "definitionURL"))
    let href = ak.vector_at(attributes, 2)
    assert(test_fly_is(href.local_name, "href"))
    assert(test_fly_is(href.prefix else trap("prefix"), "xlink"))
    assert(dom_node_optional_fly_string_is(href.namespace_, namespace_x_link))
    let xmlns = ak.vector_at(attributes, 3)
    assert(test_fly_is(xmlns.local_name, "xmlns") and xmlns.prefix == none)
    assert(dom_node_optional_fly_string_is(xmlns.namespace_, namespace_xmlns))

# mark: Serialization ===========================================================================

test "HTMLParser: escaping a string":
    let text = ak.string_from_str("a&b<c>d\u{A0}e\"f")
    let escaped = html_escape_string_utf8_view(ak.string_code_points(&text), HtmlAttributeMode.no)
    assert(test_string_is(escaped, "a&amp;b&lt;c&gt;d&nbsp;e\"f"))
    let attribute = html_escape_string_utf8_view(ak.string_code_points(&text), HtmlAttributeMode.yes)
    assert(test_string_is(attribute, "a&amp;b&lt;c&gt;d&nbsp;e&quot;f"))
    let utf16 = ak.utf16_string_from_str("x<y")
    assert(test_string_is(html_escape_string_utf16_view(ak.utf16_string_base_utf16_view(&utf16.utf16_string_base), HtmlAttributeMode.no), "x&lt;y"))

test "HTMLParser::serialize_html_fragment serializes a hand-built tree":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)

    let div = parser_test_html(&world, "div")
    test_link_child(div.dom_node(), test_text(world.realm, world.document, "a & b < c > d\u{A0}e \"q\"").dom_node())
    test_link_child(div.dom_node(), parser_test_html(&world, "br").dom_node())
    let span = parser_test_html(&world, "span")
    test_link_child(span.dom_node(), test_text(world.realm, world.document, "t").dom_node())
    test_link_child(div.dom_node(), span.dom_node())
    test_link_child(div.dom_node(), realm_create_dom_comment(world.realm, world.document, ak.utf16_string_from_str("c")).dom_node())
    let style = parser_test_html(&world, "style")
    test_link_child(style.dom_node(), test_text(world.realm, world.document, "x<y&z").dom_node())
    test_link_child(div.dom_node(), style.dom_node())
    test_link_child(div.dom_node(), parser_test_foreign(&world, "circle", namespace_svg).dom_node())
    test_link_child(div.dom_node(), parser_test_foreign(&world, "thing", parser_test_namespace()).dom_node())

    var no_shadow_roots: ak.Vector[DomShadowRoot*?]
    let inner = html_parser_serialize_html_fragment(div.dom_node(), HtmlParserSerializableShadowRoots.no, no_shadow_roots)
    assert(test_string_is(inner, "a &amp; b &lt; c &gt; d&nbsp;e \"q\"<br><span>t</span><!--c--><style>x<y&z</style><circle></circle><thing></thing>"))
    let outer = html_parser_serialize_html_fragment(span.dom_node(), HtmlParserSerializableShadowRoots.no, no_shadow_roots, DomFragmentSerializationMode.outer)
    assert(test_string_is(outer, "<span>t</span>"))
    parser_test_expect_dump(parser_test_dump(div.dom_node()), "| \"a & b < c > d\u{A0}e \"q\"\"\n| <br>\n| <span>\n|   \"t\"\n| <!-- c -->\n| <style>\n|   \"x<y&z\"\n| <svg circle>\n| <{urn:x-test} thing>\n")
    # A void element serializes as nothing.
    let br = html_parser_serialize_html_fragment(parser_test_html(&world, "br").dom_node(), HtmlParserSerializableShadowRoots.no, no_shadow_roots)
    assert(test_string_is(br, ""))

# mark: Dimension values ========================================================================

test "parse_dimension_value fails without a leading digit":
    let failures: str[7] = ["", "   ", "abc", "-5", "+5", ".5", "\u{A0}5"]
    for input in failures:
        assert(html_parse_dimension_value(ak.sv(input)) == none)
        assert(html_parse_nonzero_dimension_value(ak.sv(input)) == none)
    # Invalid UTF-8 is a failure too.
    assert(html_parse_dimension_value(ak.string_view_from_bytes(b"\xFF5")) == none)

# mark: Creating a parser =======================================================================

## The document's encoding as text ("-" for none).
func parser_test_document_encoding(document: DomDocument*) -> ak.String:
    return dom_document_encoding(document) else ak.string_from_str("-")

test "HTMLParser: the constructors hook the parser up to its document and tokenizer":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)
    let document = test_document_in_realm(world.realm)

    let parser = html_parser_create(document, ak.sv("<p>hi"), HtmlParserScriptingMode.disabled, ak.sv("latin1"))
    assert(dom_document_parser(document) == parser)
    # The encoding label is standardized.
    assert(test_string_is(parser_test_document_encoding(document), "windows-1252"))
    assert(html_parser_insertion_mode(parser) == HtmlParserInsertionMode.initial)
    assert(parser.m_frameset_ok and not parser.m_script_created)
    assert(html_parser_tokenizer(parser) == &parser.m_tokenizer)
    assert(test_string_is(html_syntax.html_tokenizer_source(&parser.m_tokenizer), "<p>hi"))

    let scripted = html_parser_create_for_scripting(document)
    assert(scripted.m_script_created and dom_document_parser(document) == scripted)
    let open_stream = html_parser_create_with_open_input_stream(document)
    assert(not open_stream.m_script_created and dom_document_parser(document) == open_stream)

test "HTMLParser::create_with_uncertain_encoding sniffs the encoding unless the document has one":
    var world = parser_test_world()
    defer web_test_destroy_vm(world.vm)

    let document = test_document_in_realm(world.realm)
    let bytes = ak.byte_buffer_copy_span_u8(b"<meta charset=koi8-r><p>x") else trap("bytes")
    _ = html_parser_create_with_uncertain_encoding(document, bytes)
    assert(test_string_is(parser_test_document_encoding(document), "KOI8-R"))

    # A transport-layer charset wins over the prescan.
    let transported = test_document_in_realm(world.realm)
    _ = html_parser_create_with_uncertain_encoding(transported, bytes, mime_type_parse(ak.sv("text/html; charset=gbk")))
    assert(test_string_is(parser_test_document_encoding(transported), "GBK"))

    # A document that already has an encoding keeps it.
    let decided = test_document_in_realm(world.realm)
    dom_document_set_encoding(decided, ak.string_from_str("UTF-8"))
    _ = html_parser_create_with_uncertain_encoding(decided, bytes)
    assert(test_string_is(parser_test_document_encoding(decided), "UTF-8"))