dymokomi/luce-browser-html
Source of release 0.5.1
0.5.1 / src / html_syntax / tests_html_tokenizer.lucb
#==============================================================================================
#
# tests_html_tokenizer - Tests/LibWeb/TestHTMLTokenizer.cpp
#
# DESCRIPTION:
# Port of Tests/LibWeb/TestHTMLTokenizer.cpp (Ladybird 47c82b38d0, lines 1-277): all 16
# test cases. The regression case hashes the tokens of Tests/LibWeb/tokenizer-test.html,
# whose bytes are `tokenizer_test_html` below. The EXPECT_* macros are functions over a
# TokenCursor (BEGIN_ENUMERATION); each test body runs under `with memory.heap` so that
# interned strings outlive the test's allocator.
#
#==============================================================================================
# mark: The harness =============================================================================
# run_tokenizer: every token of `input` tokenized as UTF-8.
func run_tokenizer(input: str) -> ak.Vector[HtmlToken]:
var tokens: ak.Vector[HtmlToken]
var tokenizer: HtmlTokenizer
html_tokenizer_construct_string_view(&tokenizer, ak.sv(input), ak.byte_string_from_str("UTF-8"))
while true:
let maybe_token = html_tokenizer_next_token(&tokenizer)
if maybe_token == none:
break
ak.vector_append_t(&tokens, maybe_token else trap("unreachable"))
return tokens
# FIXME: It's not very nice to rely on the format of HTMLToken::to_string() to stay the same.
func hash_tokens(tokens: ak.Vector[HtmlToken]*) -> u32:
var builder = ak.string_builder_new()
for token in &ak.vector_span(tokens):
let text = html_token_to_string(token)
ak.string_builder_append_string_view(&builder, ak.string_bytes_as_string_view(&text))
let view = ak.string_builder_string_view(&builder)
return ak.string_view_hash(&view)
# BEGIN_ENUMERATION(tokens): the current token and the last one passed.
struct TokenCursor:
var tokens: ak.Vector[HtmlToken]*
var index: usize
var last_token: HtmlToken*?
func begin_enumeration(tokens: ak.Vector[HtmlToken]*) -> TokenCursor:
return TokenCursor(tokens = tokens, index = 0, last_token = none)
func current_token(cursor: TokenCursor*) -> HtmlToken*:
assert(cursor.index < ak.vector_size(cursor.tokens))
return ak.vector_at(cursor.tokens, cursor.index)
# NEXT_TOKEN()
func next_token(cursor: TokenCursor*):
cursor.last_token = current_token(cursor)
cursor.index += 1
# END_ENUMERATION()
func end_enumeration(cursor: TokenCursor*):
assert(cursor.index == ak.vector_size(cursor.tokens))
func tag_name_is(token: const HtmlToken*, tag_name: str) -> bool:
let name = html_token_tag_name(token)
return ak.fly_string_eq_string_view(&name, ak.sv(tag_name))
# EXPECT_START_TAG_TOKEN(tag_name, start_column, end_column)
func expect_start_tag_token(cursor: TokenCursor*, tag_name: str, start_column: usize, end_column: usize):
let token = current_token(cursor)
assert(html_token_type(token) == HtmlTokenType.start_tag)
assert(tag_name_is(token, tag_name))
assert(html_token_start_position(token).column == start_column)
assert(html_token_end_position(token).column == end_column)
next_token(cursor)
# EXPECT_END_TAG_TOKEN(tag_name, start_column, end_column)
func expect_end_tag_token(cursor: TokenCursor*, tag_name: str, start_column: usize, end_column: usize):
let token = current_token(cursor)
assert(html_token_type(token) == HtmlTokenType.end_tag)
assert(tag_name_is(token, tag_name))
assert(html_token_start_position(token).column == start_column)
assert(html_token_end_position(token).column == end_column)
next_token(cursor)
# EXPECT_END_OF_FILE_TOKEN()
func expect_end_of_file_token(cursor: TokenCursor*):
assert(html_token_type(current_token(cursor)) == HtmlTokenType.end_of_file)
next_token(cursor)
# EXPECT_CHARACTER_TOKEN(character)
func expect_character_token(cursor: TokenCursor*, character: u32):
let token = current_token(cursor)
assert(html_token_type(token) == HtmlTokenType.character)
assert(html_token_code_point(token) == character)
next_token(cursor)
# EXPECT_CHARACTER_TOKENS(string)
func expect_character_tokens(cursor: TokenCursor*, string: str):
for c_ in string.bytes:
expect_character_token(cursor, c_)
# EXPECT_COMMENT_TOKEN()
func expect_comment_token(cursor: TokenCursor*):
assert(html_token_type(current_token(cursor)) == HtmlTokenType.comment)
next_token(cursor)
# EXPECT_DOCTYPE_TOKEN()
func expect_doctype_token(cursor: TokenCursor*):
assert(html_token_type(current_token(cursor)) == HtmlTokenType.doctype)
next_token(cursor)
# EXPECT_TAG_TOKEN_ATTRIBUTE(name, attribute_value, name_start_column, name_end_column, value_start_column, value_end_column)
func expect_tag_token_attribute(cursor: TokenCursor*, name: str, attribute_value: str, name_start_column: usize, name_end_column: usize, value_start_column: usize, value_end_column: usize):
let last_token = cursor.last_token else trap("VERIFY: last_token")
let attribute = html_token_raw_attribute(last_token, ak.fly_string_from_str(name)) else trap("VERIFY: attr.has_value()")
assert(ak.string_eq_string_view(&attribute.value, ak.sv(attribute_value)))
assert(attribute.name_start_position.column == name_start_column)
assert(attribute.name_end_position.column == name_end_column)
assert(attribute.value_start_position.column == value_start_column)
assert(attribute.value_end_position.column == value_end_column)
# EXPECT_TAG_TOKEN_ATTRIBUTE_COUNT(count)
func expect_tag_token_attribute_count(cursor: TokenCursor*, count: usize):
let last_token = cursor.last_token else trap("VERIFY: last_token")
assert(html_token_attribute_count(last_token) == count)
# mark: TestHTMLTokenizer.cpp ===================================================================
test "empty":
with memory.heap:
var tokens = run_tokenizer("")
var cursor = begin_enumeration(&tokens)
expect_end_of_file_token(&cursor)
end_enumeration(&cursor)
test "basic":
with memory.heap:
var tokens = run_tokenizer("<html><head></head><body></body></html>")
var cursor = begin_enumeration(&tokens)
expect_start_tag_token(&cursor, "html", 1, 5)
expect_start_tag_token(&cursor, "head", 7, 11)
expect_end_tag_token(&cursor, "head", 14, 18)
expect_start_tag_token(&cursor, "body", 20, 24)
expect_end_tag_token(&cursor, "body", 27, 31)
expect_end_tag_token(&cursor, "html", 34, 38)
expect_end_of_file_token(&cursor)
end_enumeration(&cursor)
test "basic_with_text":
with memory.heap:
var tokens = run_tokenizer("<p>This is some text.</p>")
var cursor = begin_enumeration(&tokens)
expect_start_tag_token(&cursor, "p", 1, 2)
expect_character_tokens(&cursor, "This is some text.")
expect_end_tag_token(&cursor, "p", 23, 24)
expect_end_of_file_token(&cursor)
end_enumeration(&cursor)
test "unquoted_attributes":
with memory.heap:
var tokens = run_tokenizer("<p foo=bar>")
var cursor = begin_enumeration(&tokens)
expect_start_tag_token(&cursor, "p", 1, 10)
expect_tag_token_attribute_count(&cursor, 1)
expect_tag_token_attribute(&cursor, "foo", "bar", 3, 6, 7, 10)
expect_end_of_file_token(&cursor)
end_enumeration(&cursor)
test "single_quoted_attributes":
with memory.heap:
var tokens = run_tokenizer("<p foo='bar'>")
var cursor = begin_enumeration(&tokens)
expect_start_tag_token(&cursor, "p", 1, 12)
expect_tag_token_attribute_count(&cursor, 1)
expect_tag_token_attribute(&cursor, "foo", "bar", 3, 6, 7, 12)
expect_end_of_file_token(&cursor)
end_enumeration(&cursor)
test "double_quoted_attributes":
with memory.heap:
var tokens = run_tokenizer("<p foo=\"bar\">")
var cursor = begin_enumeration(&tokens)
expect_start_tag_token(&cursor, "p", 1, 12)
expect_tag_token_attribute_count(&cursor, 1)
expect_tag_token_attribute(&cursor, "foo", "bar", 3, 6, 7, 12)
expect_end_of_file_token(&cursor)
end_enumeration(&cursor)
test "valueless_attribute":
with memory.heap:
var tokens = run_tokenizer("<p foo>")
var cursor = begin_enumeration(&tokens)
expect_start_tag_token(&cursor, "p", 1, 6)
expect_tag_token_attribute_count(&cursor, 1)
expect_tag_token_attribute(&cursor, "foo", "", 3, 6, 0, 0)
expect_end_of_file_token(&cursor)
end_enumeration(&cursor)
test "multiple_attributes":
with memory.heap:
var tokens = run_tokenizer("<p foo=\"bar\" baz=foobar biz foo2=\"bar2\">")
var cursor = begin_enumeration(&tokens)
expect_start_tag_token(&cursor, "p", 1, 39)
expect_tag_token_attribute_count(&cursor, 4)
expect_tag_token_attribute(&cursor, "foo", "bar", 3, 6, 7, 12)
expect_tag_token_attribute(&cursor, "baz", "foobar", 13, 16, 17, 23)
expect_tag_token_attribute(&cursor, "biz", "", 24, 27, 0, 0)
expect_tag_token_attribute(&cursor, "foo2", "bar2", 28, 32, 33, 39)
expect_end_of_file_token(&cursor)
end_enumeration(&cursor)
test "character_reference_in_attribute":
with memory.heap:
var tokens = run_tokenizer("<p foo=a&b bar='a&b' baz=\"a&b\">")
var cursor = begin_enumeration(&tokens)
expect_start_tag_token(&cursor, "p", 1, 43)
expect_tag_token_attribute_count(&cursor, 3)
expect_tag_token_attribute(&cursor, "foo", "a&b", 3, 6, 7, 14)
expect_tag_token_attribute(&cursor, "bar", "a&b", 15, 18, 19, 28)
expect_tag_token_attribute(&cursor, "baz", "a&b", 29, 32, 33, 43)
expect_end_of_file_token(&cursor)
end_enumeration(&cursor)
test "named_character_reference":
with memory.heap:
var tokens = run_tokenizer("⋶¬it;&cz")
var cursor = begin_enumeration(&tokens)
expect_character_token(&cursor, 0x22F6) # ⋶
expect_character_token(&cursor, 0xAC) # ¬ (backtracked from ¬it)
expect_character_tokens(&cursor, "it")
expect_character_token(&cursor, (u32)';')
expect_character_tokens(&cursor, "&cz") # invalid
expect_end_of_file_token(&cursor)
end_enumeration(&cursor)
test "numeric_character_reference":
with memory.heap:
var tokens = run_tokenizer("ї")
var cursor = begin_enumeration(&tokens)
expect_character_token(&cursor, 1111)
expect_end_of_file_token(&cursor)
end_enumeration(&cursor)
test "hex_character_reference":
with memory.heap:
var tokens = run_tokenizer("ꄫZ")
var cursor = begin_enumeration(&tokens)
expect_character_token(&cursor, 0xA12B)
expect_character_token(&cursor, (u32)'Z')
expect_end_of_file_token(&cursor)
end_enumeration(&cursor)
test "comment":
with memory.heap:
var tokens = run_tokenizer("<p><!-- This is a comment --></p>")
var cursor = begin_enumeration(&tokens)
expect_start_tag_token(&cursor, "p", 1, 2)
expect_comment_token(&cursor)
expect_end_tag_token(&cursor, "p", 31, 32)
expect_end_of_file_token(&cursor)
end_enumeration(&cursor)
test "doctype":
with memory.heap:
var tokens = run_tokenizer("<!DOCTYPE html><html></html>")
var cursor = begin_enumeration(&tokens)
expect_doctype_token(&cursor)
expect_start_tag_token(&cursor, "html", 16, 20)
expect_end_tag_token(&cursor, "html", 23, 27)
# The bytes of Tests/LibWeb/tokenizer-test.html (594 bytes).
let tokenizer_test_html: str = "<!DOCTYPE html>\n<html lang=\"en\">\n<head>\n <meta charset=\"UTF-8\">\n <title>This is a test page :^)</title>\n <script>\n let foo = 2;\n var bar = 3;\n if (foo < bar)\n alert(\"Check happens with HTML special <characters>.\");\n let regex = /</g;\n </script>\n</head>\n<body>\n <p>This is the first paragraph.</p>\n <p foo=\"bar\">The second paragraph has an attribute!</p>\n <!-- This is a standard HTML comment. -->\n <!----- This comment has a few too many dashes. ----->\n <!----> <!-- <= There is an empty comment over there. -->\n</body>\n</html>\n"
test "regression":
with memory.heap:
# NOTE: This relies on the format of HTMLToken::to_string() staying the same.
# If that changes, or something is added to the test HTML, the hash needs to be adjusted.
assert(tokenizer_test_html.length == 594)
var tokens = run_tokenizer(tokenizer_test_html)
let tokens_hash = hash_tokens(&tokens)
assert(tokens_hash == 3657343287)
test "ambiguous_ampersand_offset":
with memory.heap:
var tokens = run_tokenizer("&a")
let token = ak.vector_first(&tokens)
assert(html_token_type(token) == HtmlTokenType.character)
assert(html_token_start_position(token).line == 0)
assert(html_token_start_position(token).column == 1)