Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-html

Source of release 0.5.1

0.5.1 / src / html_syntax / tests_html_tokenizer.lucb

#==============================================================================================
#
#   tests_html_tokenizer - Tests/LibWeb/TestHTMLTokenizer.cpp
#
#   DESCRIPTION:
#       Port of Tests/LibWeb/TestHTMLTokenizer.cpp (Ladybird 47c82b38d0, lines 1-277): all 16
#       test cases. The regression case hashes the tokens of Tests/LibWeb/tokenizer-test.html,
#       whose bytes are `tokenizer_test_html` below. The EXPECT_* macros are functions over a
#       TokenCursor (BEGIN_ENUMERATION); each test body runs under `with memory.heap` so that
#       interned strings outlive the test's allocator.
#
#==============================================================================================

# mark: The harness =============================================================================

# run_tokenizer: every token of `input` tokenized as UTF-8.
func run_tokenizer(input: str) -> ak.Vector[HtmlToken]:
    var tokens: ak.Vector[HtmlToken]
    var tokenizer: HtmlTokenizer
    html_tokenizer_construct_string_view(&tokenizer, ak.sv(input), ak.byte_string_from_str("UTF-8"))
    while true:
        let maybe_token = html_tokenizer_next_token(&tokenizer)
        if maybe_token == none:
            break
        ak.vector_append_t(&tokens, maybe_token else trap("unreachable"))
    return tokens

# FIXME: It's not very nice to rely on the format of HTMLToken::to_string() to stay the same.
func hash_tokens(tokens: ak.Vector[HtmlToken]*) -> u32:
    var builder = ak.string_builder_new()
    for token in &ak.vector_span(tokens):
        let text = html_token_to_string(token)
        ak.string_builder_append_string_view(&builder, ak.string_bytes_as_string_view(&text))
    let view = ak.string_builder_string_view(&builder)
    return ak.string_view_hash(&view)

# BEGIN_ENUMERATION(tokens): the current token and the last one passed.
struct TokenCursor:
    var tokens: ak.Vector[HtmlToken]*
    var index: usize
    var last_token: HtmlToken*?

func begin_enumeration(tokens: ak.Vector[HtmlToken]*) -> TokenCursor:
    return TokenCursor(tokens = tokens, index = 0, last_token = none)

func current_token(cursor: TokenCursor*) -> HtmlToken*:
    assert(cursor.index < ak.vector_size(cursor.tokens))
    return ak.vector_at(cursor.tokens, cursor.index)

# NEXT_TOKEN()
func next_token(cursor: TokenCursor*):
    cursor.last_token = current_token(cursor)
    cursor.index += 1

# END_ENUMERATION()
func end_enumeration(cursor: TokenCursor*):
    assert(cursor.index == ak.vector_size(cursor.tokens))

func tag_name_is(token: const HtmlToken*, tag_name: str) -> bool:
    let name = html_token_tag_name(token)
    return ak.fly_string_eq_string_view(&name, ak.sv(tag_name))

# EXPECT_START_TAG_TOKEN(tag_name, start_column, end_column)
func expect_start_tag_token(cursor: TokenCursor*, tag_name: str, start_column: usize, end_column: usize):
    let token = current_token(cursor)
    assert(html_token_type(token) == HtmlTokenType.start_tag)
    assert(tag_name_is(token, tag_name))
    assert(html_token_start_position(token).column == start_column)
    assert(html_token_end_position(token).column == end_column)
    next_token(cursor)

# EXPECT_END_TAG_TOKEN(tag_name, start_column, end_column)
func expect_end_tag_token(cursor: TokenCursor*, tag_name: str, start_column: usize, end_column: usize):
    let token = current_token(cursor)
    assert(html_token_type(token) == HtmlTokenType.end_tag)
    assert(tag_name_is(token, tag_name))
    assert(html_token_start_position(token).column == start_column)
    assert(html_token_end_position(token).column == end_column)
    next_token(cursor)

# EXPECT_END_OF_FILE_TOKEN()
func expect_end_of_file_token(cursor: TokenCursor*):
    assert(html_token_type(current_token(cursor)) == HtmlTokenType.end_of_file)
    next_token(cursor)

# EXPECT_CHARACTER_TOKEN(character)
func expect_character_token(cursor: TokenCursor*, character: u32):
    let token = current_token(cursor)
    assert(html_token_type(token) == HtmlTokenType.character)
    assert(html_token_code_point(token) == character)
    next_token(cursor)

# EXPECT_CHARACTER_TOKENS(string)
func expect_character_tokens(cursor: TokenCursor*, string: str):
    for c_ in string.bytes:
        expect_character_token(cursor, c_)

# EXPECT_COMMENT_TOKEN()
func expect_comment_token(cursor: TokenCursor*):
    assert(html_token_type(current_token(cursor)) == HtmlTokenType.comment)
    next_token(cursor)

# EXPECT_DOCTYPE_TOKEN()
func expect_doctype_token(cursor: TokenCursor*):
    assert(html_token_type(current_token(cursor)) == HtmlTokenType.doctype)
    next_token(cursor)

# EXPECT_TAG_TOKEN_ATTRIBUTE(name, attribute_value, name_start_column, name_end_column, value_start_column, value_end_column)
func expect_tag_token_attribute(cursor: TokenCursor*, name: str, attribute_value: str, name_start_column: usize, name_end_column: usize, value_start_column: usize, value_end_column: usize):
    let last_token = cursor.last_token else trap("VERIFY: last_token")
    let attribute = html_token_raw_attribute(last_token, ak.fly_string_from_str(name)) else trap("VERIFY: attr.has_value()")
    assert(ak.string_eq_string_view(&attribute.value, ak.sv(attribute_value)))
    assert(attribute.name_start_position.column == name_start_column)
    assert(attribute.name_end_position.column == name_end_column)
    assert(attribute.value_start_position.column == value_start_column)
    assert(attribute.value_end_position.column == value_end_column)

# EXPECT_TAG_TOKEN_ATTRIBUTE_COUNT(count)
func expect_tag_token_attribute_count(cursor: TokenCursor*, count: usize):
    let last_token = cursor.last_token else trap("VERIFY: last_token")
    assert(html_token_attribute_count(last_token) == count)

# mark: TestHTMLTokenizer.cpp ===================================================================

test "empty":
    with memory.heap:
        var tokens = run_tokenizer("")
        var cursor = begin_enumeration(&tokens)
        expect_end_of_file_token(&cursor)
        end_enumeration(&cursor)

test "basic":
    with memory.heap:
        var tokens = run_tokenizer("<html><head></head><body></body></html>")
        var cursor = begin_enumeration(&tokens)
        expect_start_tag_token(&cursor, "html", 1, 5)
        expect_start_tag_token(&cursor, "head", 7, 11)
        expect_end_tag_token(&cursor, "head", 14, 18)
        expect_start_tag_token(&cursor, "body", 20, 24)
        expect_end_tag_token(&cursor, "body", 27, 31)
        expect_end_tag_token(&cursor, "html", 34, 38)
        expect_end_of_file_token(&cursor)
        end_enumeration(&cursor)

test "basic_with_text":
    with memory.heap:
        var tokens = run_tokenizer("<p>This is some text.</p>")
        var cursor = begin_enumeration(&tokens)
        expect_start_tag_token(&cursor, "p", 1, 2)
        expect_character_tokens(&cursor, "This is some text.")
        expect_end_tag_token(&cursor, "p", 23, 24)
        expect_end_of_file_token(&cursor)
        end_enumeration(&cursor)

test "unquoted_attributes":
    with memory.heap:
        var tokens = run_tokenizer("<p foo=bar>")
        var cursor = begin_enumeration(&tokens)
        expect_start_tag_token(&cursor, "p", 1, 10)
        expect_tag_token_attribute_count(&cursor, 1)
        expect_tag_token_attribute(&cursor, "foo", "bar", 3, 6, 7, 10)
        expect_end_of_file_token(&cursor)
        end_enumeration(&cursor)

test "single_quoted_attributes":
    with memory.heap:
        var tokens = run_tokenizer("<p foo='bar'>")
        var cursor = begin_enumeration(&tokens)
        expect_start_tag_token(&cursor, "p", 1, 12)
        expect_tag_token_attribute_count(&cursor, 1)
        expect_tag_token_attribute(&cursor, "foo", "bar", 3, 6, 7, 12)
        expect_end_of_file_token(&cursor)
        end_enumeration(&cursor)

test "double_quoted_attributes":
    with memory.heap:
        var tokens = run_tokenizer("<p foo=\"bar\">")
        var cursor = begin_enumeration(&tokens)
        expect_start_tag_token(&cursor, "p", 1, 12)
        expect_tag_token_attribute_count(&cursor, 1)
        expect_tag_token_attribute(&cursor, "foo", "bar", 3, 6, 7, 12)
        expect_end_of_file_token(&cursor)
        end_enumeration(&cursor)

test "valueless_attribute":
    with memory.heap:
        var tokens = run_tokenizer("<p foo>")
        var cursor = begin_enumeration(&tokens)
        expect_start_tag_token(&cursor, "p", 1, 6)
        expect_tag_token_attribute_count(&cursor, 1)
        expect_tag_token_attribute(&cursor, "foo", "", 3, 6, 0, 0)
        expect_end_of_file_token(&cursor)
        end_enumeration(&cursor)

test "multiple_attributes":
    with memory.heap:
        var tokens = run_tokenizer("<p foo=\"bar\" baz=foobar biz foo2=\"bar2\">")
        var cursor = begin_enumeration(&tokens)
        expect_start_tag_token(&cursor, "p", 1, 39)
        expect_tag_token_attribute_count(&cursor, 4)
        expect_tag_token_attribute(&cursor, "foo", "bar", 3, 6, 7, 12)
        expect_tag_token_attribute(&cursor, "baz", "foobar", 13, 16, 17, 23)
        expect_tag_token_attribute(&cursor, "biz", "", 24, 27, 0, 0)
        expect_tag_token_attribute(&cursor, "foo2", "bar2", 28, 32, 33, 39)
        expect_end_of_file_token(&cursor)
        end_enumeration(&cursor)

test "character_reference_in_attribute":
    with memory.heap:
        var tokens = run_tokenizer("<p foo=a&amp;b bar='a&#38;b' baz=\"a&#x26;b\">")
        var cursor = begin_enumeration(&tokens)
        expect_start_tag_token(&cursor, "p", 1, 43)
        expect_tag_token_attribute_count(&cursor, 3)
        expect_tag_token_attribute(&cursor, "foo", "a&b", 3, 6, 7, 14)
        expect_tag_token_attribute(&cursor, "bar", "a&b", 15, 18, 19, 28)
        expect_tag_token_attribute(&cursor, "baz", "a&b", 29, 32, 33, 43)
        expect_end_of_file_token(&cursor)
        end_enumeration(&cursor)

test "named_character_reference":
    with memory.heap:
        var tokens = run_tokenizer("&notinvc;&notit;&cz")
        var cursor = begin_enumeration(&tokens)
        expect_character_token(&cursor, 0x22F6)  # &notinvc;
        expect_character_token(&cursor, 0xAC)  # &not (backtracked from &notit)
        expect_character_tokens(&cursor, "it")
        expect_character_token(&cursor, (u32)';')
        expect_character_tokens(&cursor, "&cz")  # invalid
        expect_end_of_file_token(&cursor)
        end_enumeration(&cursor)

test "numeric_character_reference":
    with memory.heap:
        var tokens = run_tokenizer("&#1111")
        var cursor = begin_enumeration(&tokens)
        expect_character_token(&cursor, 1111)
        expect_end_of_file_token(&cursor)
        end_enumeration(&cursor)

test "hex_character_reference":
    with memory.heap:
        var tokens = run_tokenizer("&#xA12bZ")
        var cursor = begin_enumeration(&tokens)
        expect_character_token(&cursor, 0xA12B)
        expect_character_token(&cursor, (u32)'Z')
        expect_end_of_file_token(&cursor)
        end_enumeration(&cursor)

test "comment":
    with memory.heap:
        var tokens = run_tokenizer("<p><!-- This is a comment --></p>")
        var cursor = begin_enumeration(&tokens)
        expect_start_tag_token(&cursor, "p", 1, 2)
        expect_comment_token(&cursor)
        expect_end_tag_token(&cursor, "p", 31, 32)
        expect_end_of_file_token(&cursor)
        end_enumeration(&cursor)

test "doctype":
    with memory.heap:
        var tokens = run_tokenizer("<!DOCTYPE html><html></html>")
        var cursor = begin_enumeration(&tokens)
        expect_doctype_token(&cursor)
        expect_start_tag_token(&cursor, "html", 16, 20)
        expect_end_tag_token(&cursor, "html", 23, 27)

# The bytes of Tests/LibWeb/tokenizer-test.html (594 bytes).
let tokenizer_test_html: str = "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n    <meta charset=\"UTF-8\">\n    <title>This is a test page :^)</title>\n    <script>\n        let foo = 2;\n        var bar = 3;\n        if (foo < bar)\n            alert(\"Check happens with HTML special <characters>.\");\n        let regex = /</g;\n    </script>\n</head>\n<body>\n    <p>This is the first paragraph.</p>\n    <p foo=\"bar\">The second paragraph has an attribute!</p>\n    <!-- This is a standard HTML comment. -->\n    <!----- This comment has a few too many dashes. ----->\n    <!----> <!-- <= There is an empty comment over there. -->\n</body>\n</html>\n"

test "regression":
    with memory.heap:
        # NOTE: This relies on the format of HTMLToken::to_string() staying the same.
        #       If that changes, or something is added to the test HTML, the hash needs to be adjusted.
        assert(tokenizer_test_html.length == 594)
        var tokens = run_tokenizer(tokenizer_test_html)
        let tokens_hash = hash_tokens(&tokens)
        assert(tokens_hash == 3657343287)

test "ambiguous_ampersand_offset":
    with memory.heap:
        var tokens = run_tokenizer("&a")
        let token = ak.vector_first(&tokens)
        assert(html_token_type(token) == HtmlTokenType.character)
        assert(html_token_start_position(token).line == 0)
        assert(html_token_start_position(token).column == 1)