Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-html

Source of release 0.5.1

0.5.1 / src / html_syntax / tests_html_tokenizer_cases_1.lucb

#==============================================================================================
#
#   tests_html_tokenizer_cases_1 - token dumps pinned to the donor (part 1 of 2)
#
#   DESCRIPTION:
#       Cases for every tokenizer state, with the expected tokens printed by the donor's
#       HTMLTokenizer (reference build, Ladybird 47c82b38d0) through a local oracle
#       (.donors/luce-browser-tools/html_tokenizer_oracle). check_case is in
#       tests_html_tokenizer_cases_1. Generated from the oracle's output; regenerate rather
#       than edit.
#
#==============================================================================================

# mark: The dump ================================================================================

# The oracle's escape_into: \n, \r, \t, \0 and \\ spelled out, every other byte as it is.
func dump_escaped(builder: ak.StringBuilder*, view: ak.StringView):
    for byte in ak.string_view_bytes(&view):
        if byte == '\n':
            ak.string_builder_append(builder, "\\n")
        elif byte == '\r':
            ak.string_builder_append(builder, "\\r")
        elif byte == '\t':
            ak.string_builder_append(builder, "\\t")
        elif byte == 0:
            ak.string_builder_append(builder, "\\0")
        elif byte == '\\':
            ak.string_builder_append(builder, "\\\\")
        else:
            ak.string_builder_append_u8(builder, byte)

func dump_escaped_string(builder: ak.StringBuilder*, string: ak.String):
    dump_escaped(builder, ak.string_bytes_as_string_view(&string))

func dump_escaped_fly_string(builder: ak.StringBuilder*, string: ak.FlyString):
    dump_escaped(builder, ak.fly_string_bytes_as_string_view(&string))

# appendff("{}:{}", position.line, position.column)
func dump_position(builder: ak.StringBuilder*, position: HtmlTokenPosition):
    var buffer: u8[48]
    ak.string_builder_append(builder, strings.format(buffer, f"{position.line}:{position.column}") else trap("format"))

# appendff("U+{:04X}", code_point)
func dump_code_point(builder: ak.StringBuilder*, code_point: u32):
    let digits = "0123456789ABCDEF".bytes
    var text: u8[8]
    var count: usize = 0
    var rest = code_point
    while rest != 0 or count < 4:
        text[count] = digits[rest & 0xF]
        rest = rest >> 4
        count += 1
    ak.string_builder_append(builder, "U+")
    while count > 0:
        count -= 1
        ak.string_builder_append_u8(builder, text[count])

# The for_each_attribute lambda of the oracle's dump.
func dump_attribute(context: void*?, attribute: HtmlTokenAttribute*) -> ak.IterationDecision:
    let builder = (ak.StringBuilder*)(context else trap("no captures"))
    ak.string_builder_append(builder, " [")
    dump_escaped_fly_string(builder, attribute.local_name)
    ak.string_builder_append(builder, "=")
    dump_escaped_string(builder, attribute.value)
    ak.string_builder_append(builder, " ")
    dump_position(builder, attribute.name_start_position)
    ak.string_builder_append(builder, "-")
    dump_position(builder, attribute.name_end_position)
    ak.string_builder_append(builder, " ")
    dump_position(builder, attribute.value_start_position)
    ak.string_builder_append(builder, "-")
    dump_position(builder, attribute.value_end_position)
    ak.string_builder_append(builder, "]")
    return ak.IterationDecision.continue_

# One line per token, as the oracle's dump prints it.
func dump_token(builder: ak.StringBuilder*, token: HtmlToken*):
    match html_token_type(token):
        .doctype:
            let data = html_token_doctype_data(token)
            ak.string_builder_append(builder, "DOCTYPE name=")
            if data.missing_name:
                ak.string_builder_append(builder, "<missing>")
            else:
                dump_escaped_string(builder, data.name)
            ak.string_builder_append(builder, " public=")
            if data.missing_public_identifier:
                ak.string_builder_append(builder, "<missing>")
            else:
                dump_escaped_string(builder, data.public_identifier)
            ak.string_builder_append(builder, " system=")
            if data.missing_system_identifier:
                ak.string_builder_append(builder, "<missing>")
            else:
                dump_escaped_string(builder, data.system_identifier)
            ak.string_builder_append(builder, " quirks=1" if data.force_quirks else " quirks=0")
        .start_tag, .end_tag:
            ak.string_builder_append(builder, "StartTag " if html_token_is_start_tag(token) else "EndTag ")
            dump_escaped_fly_string(builder, html_token_tag_name(token))
            var buffer: u8[96]
            ak.string_builder_append(builder, strings.format(buffer, f" self_closing={(i32)html_token_is_self_closing(token)} dup={(i32)html_token_had_duplicate_attribute(token)} attributes={html_token_attribute_count(token)}") else trap("format"))
            html_token_for_each_attribute(token, ak.Function1[HtmlTokenAttribute*, ak.IterationDecision](call = dump_attribute, context = builder))
        .comment:
            ak.string_builder_append(builder, "Comment ")
            dump_escaped_string(builder, html_token_comment(token))
        .character:
            ak.string_builder_append(builder, "Character ")
            dump_code_point(builder, html_token_code_point(token))
        .end_of_file:
            ak.string_builder_append(builder, "EOF")
        _:
            ak.string_builder_append(builder, "Invalid")
    ak.string_builder_append(builder, " @")
    dump_position(builder, html_token_start_position(token))
    ak.string_builder_append(builder, "-")
    dump_position(builder, html_token_end_position(token))
    ak.string_builder_append(builder, "\n")

# The oracle's run: tokenize `input` as UTF-8, switch to `state` (unless -1) after the first
# token, stop after the EOF token, and compare the dump with `expected`.
func check_case(input: str, state: i32, expected: str):
    var tokenizer: HtmlTokenizer
    html_tokenizer_construct_string_view(&tokenizer, ak.sv(input), ak.byte_string_from_str("UTF-8"))
    var builder = ak.string_builder_new()
    var first = true
    while true:
        var token = html_tokenizer_next_token(&tokenizer) else break
        dump_token(&builder, &token)
        if html_token_is_end_of_file(&token):
            break
        if first and state != -1:
            html_tokenizer_switch_to_html_tokenizer_state(&tokenizer, (HtmlTokenizerState)state)
        first = false
    let actual = ak.string_builder_string_view(&builder)
    if not ak.string_view_equals(&actual, expected):
        print(f"input:    {input}")
        print(f"expected: {expected}")
        print(f"actual:   {(str)ak.string_view_bytes(&actual)}")
        trap("the tokens differ from the donor's")

# mark: DOCTYPE =================================================================================

test "cases_doctype_1":
    with memory.heap:
        # from the data state
        check_case("<!DOCTYPE html>", -1,
                   "DOCTYPE name=html public=<missing> system=<missing> quirks=0 @0:11-0:15\nEOF @0:15-0:15\n")
        # from the data state
        check_case("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">", -1,
                   "DOCTYPE name=html public=-//W3C//DTD XHTML 1.0 Strict//EN system=http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd quirks=0 @0:11-0:109\nEOF @0:109-0:109\n")
        # from the data state
        check_case("<!doctype html system 'about:legacy-compat'>", -1,
                   "DOCTYPE name=html public=<missing> system=about:legacy-compat quirks=0 @0:11-0:44\nEOF @0:44-0:44\n")
        # from the data state
        check_case("<!DOCTYPE>", -1,
                   "DOCTYPE name=<missing> public=<missing> system=<missing> quirks=1 @0:10-0:10\nEOF @0:10-0:10\n")
        # from the data state
        check_case("<!DOCTYPE", -1,
                   "DOCTYPE name=<missing> public=<missing> system=<missing> quirks=1 @0:9-0:0\nEOF @0:9-0:9\n")
        # from the data state
        check_case("<!DOCTYPEhtml>", -1,
                   "DOCTYPE name=html public=<missing> system=<missing> quirks=0 @0:10-0:14\nEOF @0:14-0:14\n")
        # from the data state
        check_case("<!DOCTYPE html PUBLIC>", -1,
                   "DOCTYPE name=html public=<missing> system=<missing> quirks=1 @0:11-0:22\nEOF @0:22-0:22\n")
        # from the data state
        check_case("<!DOCTYPE html PUBLIC\"x\"'y'>", -1,
                   "DOCTYPE name=html public=x system=y quirks=0 @0:11-0:28\nEOF @0:28-0:28\n")

test "cases_doctype_2":
    with memory.heap:
        # from the data state
        check_case("<!DOCTYPE html PUBLIC 'x' \"y\" z>", -1,
                   "DOCTYPE name=html public=x system=y quirks=0 @0:11-0:32\nEOF @0:32-0:32\n")
        # from the data state
        check_case("<!DOCTYPE html public \"a>", -1,
                   "DOCTYPE name=html public=a system=<missing> quirks=1 @0:11-0:25\nEOF @0:25-0:25\n")
        # from the data state
        check_case("<!DOCTYPE html bogus>", -1,
                   "DOCTYPE name=html public=<missing> system=<missing> quirks=1 @0:11-0:21\nEOF @0:21-0:21\n")
        # from the data state
        check_case("<!DOCTYPE html SYSTEM \"a\" junk\0>", -1,
                   "DOCTYPE name=html public=<missing> system=a quirks=0 @0:11-0:32\nEOF @0:32-0:32\n")
        # from the data state
        check_case("<!DOCTYPE html SYSTEM>", -1,
                   "DOCTYPE name=html public=<missing> system=<missing> quirks=1 @0:11-0:22\nEOF @0:22-0:22\n")
        # from the data state
        check_case("<!DOCTYPE html SYSTEM\"a", -1,
                   "DOCTYPE name=html public=<missing> system= quirks=1 @0:11-0:0\nEOF @0:23-0:23\n")
        # from the data state
        check_case("<!DOCTYPE HTML\0x\0>", -1,
                   "DOCTYPE name=html�x� public=<missing> system=<missing> quirks=0 @0:11-0:18\nEOF @0:18-0:18\n")
        # from the data state
        check_case("<!DOCTYPE \0>", -1,
                   "DOCTYPE name=� public=<missing> system=<missing> quirks=0 @0:11-0:12\nEOF @0:12-0:12\n")

test "cases_doctype_3":
    with memory.heap:
        # from the data state
        check_case("<!DOCTYPE html PUB", -1,
                   "DOCTYPE name=html public=<missing> system=<missing> quirks=1 @0:11-0:0\nEOF @0:18-0:18\n")
        # from the data state
        check_case("<!DOCTYPE html PUBLIC \"a\" 'b", -1,
                   "DOCTYPE name=html public=a system= quirks=1 @0:11-0:0\nEOF @0:28-0:28\n")
        # from the data state
        check_case("<!DOCTYPE html PUBLIC 'a'>", -1,
                   "DOCTYPE name=html public=a system=<missing> quirks=0 @0:11-0:26\nEOF @0:26-0:26\n")
        # from the data state
        check_case("<!DOCTYPE html PUBLIC \"a\0b\" 'c\0d'>", -1,
                   "DOCTYPE name=html public=a�b system=c�d quirks=0 @0:11-0:34\nEOF @0:34-0:34\n")

# mark: Comments and markup declarations ========================================================

test "cases_comments_and_markup_declarations_1":
    with memory.heap:
        # from the data state
        check_case("<!---->", -1,
                   "Comment  @0:0-0:7\nEOF @0:7-0:7\n")
        # from the data state
        check_case("<!-->", -1,
                   "Comment  @0:0-0:5\nEOF @0:5-0:5\n")
        # from the data state
        check_case("<!--->", -1,
                   "Comment  @0:0-0:6\nEOF @0:6-0:6\n")
        # from the data state
        check_case("<!-- a -- b --!>", -1,
                   "Comment  a -- b  @0:0-0:16\nEOF @0:16-0:16\n")
        # from the data state
        check_case("<!--<!-- nested -->", -1,
                   "Comment <!-- nested  @0:0-0:19\nEOF @0:19-0:19\n")
        # from the data state
        check_case("<!--<!--->", -1,
                   "Comment <!- @0:0-0:10\nEOF @0:10-0:10\n")
        # from the data state
        check_case("<!--<!---->", -1,
                   "Comment <!-- @0:0-0:11\nEOF @0:11-0:11\n")
        # from the data state
        check_case("<!--<!-x-->", -1,
                   "Comment <!-x @0:0-0:11\nEOF @0:11-0:11\n")

test "cases_comments_and_markup_declarations_2":
    with memory.heap:
        # from the data state
        check_case("<!--<<!--a-->", -1,
                   "Comment <<!--a @0:0-0:13\nEOF @0:13-0:13\n")
        # from the data state
        check_case("<!-- unterminated", -1,
                   "Comment  unterminated @0:0-0:17\nEOF @0:17-0:17\n")
        # from the data state
        check_case("<!--a--!b-->", -1,
                   "Comment a--!b @0:0-0:12\nEOF @0:12-0:12\n")
        # from the data state
        check_case("<!--a--!-->", -1,
                   "Comment a--! @0:0-0:11\nEOF @0:11-0:11\n")
        # from the data state
        check_case("<!--a--!", -1,
                   "Comment a @0:0-0:8\nEOF @0:8-0:8\n")
        # from the data state
        check_case("<!--a---->", -1,
                   "Comment a-- @0:0-0:10\nEOF @0:10-0:10\n")
        # from the data state
        check_case("<!--a--", -1,
                   "Comment a @0:0-0:7\nEOF @0:7-0:7\n")
        # from the data state
        check_case("<!--a-", -1,
                   "Comment a @0:0-0:6\nEOF @0:6-0:6\n")

test "cases_comments_and_markup_declarations_3":
    with memory.heap:
        # from the data state
        check_case("<!---", -1,
                   "Comment  @0:0-0:5\nEOF @0:5-0:5\n")
        # from the data state
        check_case("<!--", -1,
                   "Comment  @0:0-0:4\nEOF @0:4-0:4\n")
        # from the data state
        check_case("<!--\0-->", -1,
                   "Comment � @0:0-0:8\nEOF @0:8-0:8\n")
        # from the data state
        check_case("<?php bogus \0 ?>", -1,
                   "Comment ?php bogus � ? @0:0-0:16\nEOF @0:16-0:16\n")
        # from the data state
        check_case("<?php", -1,
                   "Comment ?php @0:0-0:5\nEOF @0:5-0:5\n")
        # from the data state
        check_case("</>", -1,
                   "EOF @0:3-0:3\n")
        # from the data state
        check_case("</3>", -1,
                   "Comment 3 @0:3-0:4\nEOF @0:4-0:4\n")
        # from the data state
        check_case("</", -1,
                   "Character U+003C @0:0-0:0\nCharacter U+002F @0:0-0:0\nEOF @0:2-0:2\n")

test "cases_comments_and_markup_declarations_4":
    with memory.heap:
        # from the data state
        check_case("<!x>", -1,
                   "Comment x @0:2-0:4\nEOF @0:4-0:4\n")
        # from the data state
        check_case("<!", -1,
                   "Comment  @0:2-0:2\nEOF @0:2-0:2\n")
        # from the data state
        check_case("<!-", -1,
                   "Comment - @0:2-0:3\nEOF @0:3-0:3\n")
        # from the data state
        check_case("<!DOC", -1,
                   "Comment DOC @0:2-0:5\nEOF @0:5-0:5\n")
        # from the data state
        check_case("<![CDATA[x]]>", -1,
                   "Comment [CDATA[x]] @0:9-0:13\nEOF @0:13-0:13\n")
        # from the data state
        check_case("<![CDA", -1,
                   "Comment [CDA @0:2-0:6\nEOF @0:6-0:6\n")

# mark: Tags and attributes =====================================================================

test "cases_tags_and_attributes_1":
    with memory.heap:
        # from the data state
        check_case("<a b=c d='e' f=\"g\" h>", -1,
                   "StartTag a self_closing=0 dup=0 attributes=4 [b=c 0:3-0:4 0:5-0:6] [d=e 0:7-0:8 0:9-0:12] [f=g 0:13-0:14 0:15-0:18] [h= 0:19-0:20 0:0-0:0] @0:1-0:20\nEOF @0:21-0:21\n")
        # from the data state
        check_case("<A HREF=X>", -1,
                   "StartTag a self_closing=0 dup=0 attributes=1 [href=X 0:3-0:7 0:8-0:9] @0:1-0:9\nEOF @0:10-0:10\n")
        # from the data state
        check_case("<br/>", -1,
                   "StartTag br self_closing=1 dup=0 attributes=0 @0:1-0:4\nEOF @0:5-0:5\n")
        # from the data state
        check_case("<br />", -1,
                   "StartTag br self_closing=1 dup=0 attributes=0 @0:1-0:5\nEOF @0:6-0:6\n")
        # from the data state
        check_case("<br/x>", -1,
                   "StartTag br self_closing=0 dup=0 attributes=1 [x= 0:4-0:5 0:0-0:0] @0:1-0:5\nEOF @0:6-0:6\n")
        # from the data state
        check_case("<p a=1 a=2 A=3>", -1,
                   "StartTag p self_closing=0 dup=1 attributes=1 [a=1 0:3-0:4 0:5-0:6] @0:1-0:14\nEOF @0:15-0:15\n")
        # from the data state
        check_case("<p a=\"x\"b>", -1,
                   "StartTag p self_closing=0 dup=0 attributes=2 [a=x 0:3-0:4 0:5-0:8] [b= 0:8-0:9 0:0-0:0] @0:1-0:9\nEOF @0:10-0:10\n")
        # from the data state
        check_case("<p =a>", -1,
                   "StartTag p self_closing=0 dup=0 attributes=1 [=a= 0:3-0:5 0:0-0:0] @0:1-0:5\nEOF @0:6-0:6\n")

test "cases_tags_and_attributes_2":
    with memory.heap:
        # from the data state
        check_case("<p \"a\" 'b' <c>", -1,
                   "StartTag p self_closing=0 dup=0 attributes=3 [\"a\"= 0:3-0:6 0:0-0:0] ['b'= 0:7-0:10 0:0-0:0] [<c= 0:11-0:13 0:0-0:0] @0:1-0:13\nEOF @0:14-0:14\n")
        # from the data state
        check_case("<a b=c`=d\"e'f<g>", -1,
                   "StartTag a self_closing=0 dup=0 attributes=1 [b=c`=d\"e'f<g 0:3-0:4 0:5-0:15] @0:1-0:15\nEOF @0:16-0:16\n")
        # from the data state
        check_case("<a\0b c\0=d\0 e='\0' f=\"\0\">", -1,
                   "StartTag a�b self_closing=0 dup=0 attributes=3 [c�=d� 0:5-0:7 0:8-0:10] [e=� 0:11-0:12 0:13-0:16] [f=� 0:17-0:18 0:19-0:22] @0:1-0:22\nEOF @0:23-0:23\n")
        # from the data state
        check_case("<a", -1,
                   "EOF @0:2-0:2\n")
        # from the data state
        check_case("<a b", -1,
                   "EOF @0:4-0:4\n")
        # from the data state
        check_case("<a b=", -1,
                   "EOF @0:5-0:5\n")
        # from the data state
        check_case("<a b=\"c", -1,
                   "EOF @0:7-0:7\n")
        # from the data state
        check_case("<a b='c", -1,
                   "EOF @0:7-0:7\n")

test "cases_tags_and_attributes_3":
    with memory.heap:
        # from the data state
        check_case("<a b=c", -1,
                   "EOF @0:6-0:6\n")
        # from the data state
        check_case("<a b=\"c\"", -1,
                   "EOF @0:8-0:8\n")
        # from the data state
        check_case("<a b/", -1,
                   "EOF @0:5-0:5\n")
        # from the data state
        check_case("<a b=\"c\"d>", -1,
                   "StartTag a self_closing=0 dup=0 attributes=2 [b=c 0:3-0:4 0:5-0:8] [d= 0:8-0:9 0:0-0:0] @0:1-0:9\nEOF @0:10-0:10\n")
        # from the data state
        check_case("</a b=c>", -1,
                   "EndTag a self_closing=0 dup=0 attributes=1 [b=c 0:4-0:5 0:6-0:7] @0:2-0:7\nEOF @0:8-0:8\n")
        # from the data state
        check_case("</A>", -1,
                   "EndTag a self_closing=0 dup=0 attributes=0 @0:2-0:3\nEOF @0:4-0:4\n")
        # from the data state
        check_case("< p>", -1,
                   "Character U+003C @0:0-0:0\nCharacter U+0020 @0:2-0:0\nCharacter U+0070 @0:3-0:0\nCharacter U+003E @0:4-0:0\nEOF @0:4-0:4\n")
        # from the data state
        check_case("<3", -1,
                   "Character U+003C @0:0-0:0\nCharacter U+0033 @0:2-0:0\nEOF @0:2-0:2\n")

test "cases_tags_and_attributes_4":
    with memory.heap:
        # from the data state
        check_case("<", -1,
                   "Character U+003C @0:0-0:0\nEOF @0:1-0:1\n")
        # from the data state
        check_case("a<b\tc=d\ne=f\u{C}g=h>", -1,
                   "Character U+0061 @0:1-0:0\nStartTag b self_closing=0 dup=0 attributes=3 [c=d 0:4-0:5 0:6-0:7] [e=f 1:0-1:1 1:2-1:3] [g=h 1:4-1:5 1:6-1:7] @0:2-1:7\nEOF @1:8-1:8\n")
        # from the data state
        check_case("<a b = \"c\" >", -1,
                   "StartTag a self_closing=0 dup=0 attributes=1 [b=c 0:3-0:5 0:7-0:10] @0:1-0:11\nEOF @0:12-0:12\n")
        # from the data state
        check_case("<a b=>", -1,
                   "StartTag a self_closing=0 dup=0 attributes=1 [b= 0:3-0:4 0:5-0:0] @0:1-0:5\nEOF @0:6-0:6\n")
        # from the data state
        check_case("<abc\0DEF>", -1,
                   "StartTag abc�def self_closing=0 dup=0 attributes=0 @0:1-0:8\nEOF @0:9-0:9\n")

# mark: Character references ====================================================================

test "cases_character_references_1":
    with memory.heap:
        # from the data state
        check_case("&amp; &AMP &amp &ampx &amp=", -1,
                   "Character U+0026 @0:5-0:0\nCharacter U+0020 @0:6-0:0\nCharacter U+0026 @0:10-0:0\nCharacter U+0020 @0:11-0:0\nCharacter U+0026 @0:15-0:0\nCharacter U+0020 @0:16-0:0\nCharacter U+0026 @0:20-0:0\nCharacter U+0078 @0:21-0:0\nCharacter U+0020 @0:22-0:0\nCharacter U+0026 @0:26-0:0\nCharacter U+003D @0:27-0:0\nEOF @0:27-0:27\n")
        # from the data state
        check_case("&notin; &notindot; &notinvc;", -1,
                   "Character U+2209 @0:7-0:0\nCharacter U+0020 @0:8-0:0\nCharacter U+22F5 @0:18-0:0\nCharacter U+0338 @0:18-0:0\nCharacter U+0020 @0:19-0:0\nCharacter U+22F6 @0:28-0:0\nEOF @0:28-0:28\n")
        # from the data state
        check_case("&NotEqualTilde; &fjlig; &nGt; &acE;", -1,
                   "Character U+2242 @0:15-0:0\nCharacter U+0338 @0:15-0:0\nCharacter U+0020 @0:16-0:0\nCharacter U+0066 @0:23-0:0\nCharacter U+006A @0:23-0:0\nCharacter U+0020 @0:24-0:0\nCharacter U+226B @0:29-0:0\nCharacter U+20D2 @0:29-0:0\nCharacter U+0020 @0:30-0:0\nCharacter U+223E @0:35-0:0\nCharacter U+0333 @0:35-0:0\nEOF @0:35-0:35\n")
        # from the data state
        check_case("&#0; &#x110000; &#xD800; &#128; &#x9F; &#13; &#x81;", -1,
                   "Character U+FFFD @0:4-0:0\nCharacter U+0020 @0:5-0:0\nCharacter U+FFFD @0:15-0:0\nCharacter U+0020 @0:16-0:0\nCharacter U+FFFD @0:24-0:0\nCharacter U+0020 @0:25-0:0\nCharacter U+20AC @0:31-0:0\nCharacter U+0020 @0:32-0:0\nCharacter U+0178 @0:38-0:0\nCharacter U+0020 @0:39-0:0\nCharacter U+000D @0:44-0:0\nCharacter U+0020 @0:45-0:0\nCharacter U+0081 @0:51-0:0\nEOF @0:51-0:51\n")
        # from the data state
        check_case("&#99999999999; &#xFFFFFFFFF;", -1,
                   "Character U+FFFD @0:14-0:0\nCharacter U+0020 @0:15-0:0\nCharacter U+FFFD @0:28-0:0\nEOF @0:28-0:28\n")
        # from the data state
        check_case("&#; &#x; &#xZ &#a &# &#x", -1,
                   "Character U+0026 @0:3-0:0\nCharacter U+0023 @0:3-0:0\nCharacter U+003B @0:3-0:0\nCharacter U+0020 @0:4-0:0\nCharacter U+0026 @0:8-0:0\nCharacter U+0023 @0:8-0:0\nCharacter U+0078 @0:8-0:0\nCharacter U+003B @0:8-0:0\nCharacter U+0020 @0:9-0:0\nCharacter U+0026 @0:13-0:0\nCharacter U+0023 @0:13-0:0\nCharacter U+0078 @0:13-0:0\nCharacter U+005A @0:13-0:0\nCharacter U+0020 @0:14-0:0\nCharacter U+0026 @0:17-0:0\nCharacter U+0023 @0:17-0:0\nCharacter U+0061 @0:17-0:0\nCharacter U+0020 @0:18-0:0\nCharacter U+0026 @0:21-0:0\nCharacter U+0023 @0:21-0:0\nCharacter U+0020 @0:21-0:0\nCharacter U+0026 @0:24-0:0\nCharacter U+0023 @0:24-0:0\nCharacter U+0078 @0:24-0:0\nEOF @0:24-0:24\n")
        # from the data state
        check_case("&unknown; &am &; & &", -1,
                   "Character U+0026 @0:1-0:0\nCharacter U+0075 @0:2-0:0\nCharacter U+006E @0:3-0:0\nCharacter U+006B @0:4-0:0\nCharacter U+006E @0:5-0:0\nCharacter U+006F @0:6-0:0\nCharacter U+0077 @0:7-0:0\nCharacter U+006E @0:8-0:0\nCharacter U+003B @0:9-0:0\nCharacter U+0020 @0:10-0:0\nCharacter U+0026 @0:11-0:0\nCharacter U+0061 @0:12-0:0\nCharacter U+006D @0:13-0:0\nCharacter U+0020 @0:14-0:0\nCharacter U+0026 @0:16-0:0\nCharacter U+003B @0:16-0:0\nCharacter U+0020 @0:17-0:0\nCharacter U+0026 @0:19-0:0\nCharacter U+0020 @0:19-0:0\nCharacter U+0026 @0:20-0:0\nEOF @0:20-0:20\n")
        # from the data state
        check_case("&#65&#x41&#X41;&#x0041;", -1,
                   "Character U+0041 @0:4-0:0\nCharacter U+0041 @0:9-0:0\nCharacter U+0041 @0:15-0:0\nCharacter U+0041 @0:23-0:0\nEOF @0:23-0:23\n")

test "cases_character_references_2":
    with memory.heap:
        # from the data state
        check_case("&#xFDD0; &#xFFFE; &#x1FFFF; &#x7F; &#x0B;", -1,
                   "Character U+FDD0 @0:8-0:0\nCharacter U+0020 @0:9-0:0\nCharacter U+FFFE @0:17-0:0\nCharacter U+0020 @0:18-0:0\nCharacter U+1FFFF @0:27-0:0\nCharacter U+0020 @0:28-0:0\nCharacter U+007F @0:34-0:0\nCharacter U+0020 @0:35-0:0\nCharacter U+000B @0:41-0:0\nEOF @0:41-0:41\n")
        # from the data state
        check_case("<a href=\"?a=1&amp=2&amp;b&copy=3&copy;&copyx&copy\">", -1,
                   "StartTag a self_closing=0 dup=0 attributes=1 [href=?a=1&amp=2&b&copy=3©&copyx© 0:3-0:7 0:8-0:50] @0:1-0:50\nEOF @0:51-0:51\n")
        # from the data state
        check_case("<a x=&notit; y=&amp z='&amp' w=\"&#65;&#x42&q;\">", -1,
                   "StartTag a self_closing=0 dup=0 attributes=4 [x=&notit; 0:3-0:4 0:5-0:10] [y=& 0:11-0:12 0:13-0:17] [z=& 0:18-0:19 0:20-0:26] [w=AB&q; 0:27-0:28 0:29-0:44] @0:1-0:44\nEOF @0:45-0:45\n")
        # from the data state
        check_case("<a x=\"&\" y=& z='&#'>", -1,
                   "StartTag a self_closing=0 dup=0 attributes=3 [x=& 0:3-0:4 0:5-0:8] [y=& 0:9-0:10 0:11-0:12] [z=&# 0:13-0:14 0:15-0:19] @0:1-0:19\nEOF @0:20-0:20\n")
        # from the data state
        check_case("&Aacute&aacute;&Aacut", -1,
                   "Character U+00C1 @0:7-0:0\nCharacter U+00E1 @0:15-0:0\nCharacter U+0026 @0:0-0:0\nCharacter U+0041 @0:0-0:0\nCharacter U+0061 @0:0-0:0\nCharacter U+0063 @0:0-0:0\nCharacter U+0075 @0:0-0:0\nCharacter U+0074 @0:0-0:0\nEOF @0:0-0:0\n")
        # from the data state
        check_case("&ThickSpace;&NotSquareSubset;&bne;", -1,
                   "Character U+205F @0:12-0:0\nCharacter U+200A @0:12-0:0\nCharacter U+228F @0:29-0:0\nCharacter U+0338 @0:29-0:0\nCharacter U+003D @0:34-0:0\nCharacter U+20E5 @0:34-0:0\nEOF @0:34-0:34\n")