dymokomi/luce-browser-html
Source of release 0.5.1
0.5.1 / src / html_syntax / tests_html_token.lucb
#==============================================================================================
#
# tests_html_token - HTMLToken's API and the tokenizer's parser hooks
#
# DESCRIPTION:
# The donor has no unit test for HTMLToken beyond TestHTMLTokenizer; these tests pin the
# members the tokenizer does not reach (the ones HTMLParser uses: make_start_tag,
# attribute lookups, the adjust_* functions, the self-closing flag, to_string), and the
# tokenizer's parser-facing members (set_parser's hook, switch_to, state_name), to the
# C++ definitions in HTMLToken.h and HTMLTokenizer.h.
#
#==============================================================================================
# mark: Helpers =================================================================================
func string_is(string: ak.String, text: str) -> bool:
return ak.string_eq_string_view(&string, ak.sv(text))
func fly_is(string: ak.FlyString, text: str) -> bool:
return ak.fly_string_eq_string_view(&string, ak.sv(text))
# A start tag token with the attributes `names`, valued "v0", "v1", ...
func start_tag_with_attributes(tag_name: str, names: const str[]) -> HtmlToken:
var token = html_token_make_start_tag(ak.fly_string_from_str(tag_name))
for (index, name) in names.indexed():
var attribute = html_token_attribute_new()
attribute.local_name = ak.fly_string_from_str(name)
var buffer: u8[16]
attribute.value = ak.string_from_str(strings.format(buffer, f"v{index}") else trap("format"))
html_token_add_attribute(&token, attribute)
return token
# The for_each_attribute callbacks: count the attributes seen, stop at a name.
struct AttributeWalk:
var seen: usize
var stop_at: str
func count_attributes_until(context: void*?, attribute: const HtmlTokenAttribute*) -> ak.IterationDecision:
let walk = (AttributeWalk*)(context else trap("no captures"))
walk.seen += 1
if ak.fly_string_eq_string_view(&attribute.local_name, ak.sv(walk.stop_at)):
return ak.IterationDecision.break_
return ak.IterationDecision.continue_
func uppercase_values(context: void*?, attribute: HtmlTokenAttribute*) -> ak.IterationDecision:
attribute.value = ak.string_to_ascii_uppercase(&attribute.value)
return ak.IterationDecision.continue_
# mark: HTMLToken ===============================================================================
test "HTMLToken: types, code points, comments and DOCTYPE data":
with memory.heap:
var character = html_token_make_character(0x41)
assert(html_token_is_character(&character) and not html_token_is_start_tag(&character))
assert(html_token_code_point(&character) == 0x41)
html_token_set_code_point(&character, 0x0D)
assert(html_token_is_parser_whitespace(&character))
let whitespace: u32[5] = [0x09, 0x0A, 0x0C, 0x0D, 0x20]
for code_point in whitespace:
html_token_set_code_point(&character, code_point)
assert(html_token_is_parser_whitespace(&character))
let not_whitespace: u32[4] = [0x0B, 0x00, 0xA0, 0x41]
for code_point in not_whitespace:
html_token_set_code_point(&character, code_point)
assert(not html_token_is_parser_whitespace(&character))
var comment: HtmlToken
html_token_construct(&comment, HtmlTokenType.comment)
assert(html_token_is_comment(&comment) and not html_token_is_parser_whitespace(&comment))
assert(string_is(html_token_comment(&comment), ""))
html_token_set_comment(&comment, ak.string_from_str("hi"))
assert(string_is(html_token_comment(&comment), "hi"))
var eof: HtmlToken
html_token_construct(&eof, HtmlTokenType.end_of_file)
assert(html_token_is_end_of_file(&eof) and html_token_type(&eof) == HtmlTokenType.end_of_file)
var doctype: HtmlToken
html_token_construct(&doctype, HtmlTokenType.doctype)
assert(html_token_is_doctype(&doctype))
let data = html_token_ensure_doctype_data(&doctype)
assert(data.missing_name and data.missing_public_identifier and data.missing_system_identifier and not data.force_quirks)
data.name = ak.string_from_str("html")
assert(html_token_ensure_doctype_data(&doctype) == data)
assert(string_is(html_token_doctype_data(&doctype).name, "html"))
test "HTMLToken: tags, attribute lookups and adjustments":
with memory.heap:
let names: str[3] = ["href", "id", "class"]
var token = start_tag_with_attributes("a", names)
assert(html_token_is_start_tag(&token) and fly_is(html_token_tag_name(&token), "a"))
assert(html_token_has_attributes(&token) and html_token_attribute_count(&token) == 3)
assert(fly_is(html_token_last_attribute_const(&token).local_name, "class"))
assert(string_is(html_token_attribute(&token, ak.fly_string_from_str("id")) else trap("no id"), "v1"))
assert(html_token_attribute(&token, ak.fly_string_from_str("title")) == none)
assert(html_token_has_attribute(&token, ak.fly_string_from_str("href")))
assert(not html_token_has_attribute(&token, ak.fly_string_from_str("HREF")))
# for_each_attribute stops at IterationDecision::Break.
var walk = AttributeWalk(seen = 0, stop_at = "id")
html_token_for_each_attribute_const(&token, ak.Function1[const HtmlTokenAttribute*, ak.IterationDecision](call = count_attributes_until, context = &walk))
assert(walk.seen == 2)
html_token_for_each_attribute(&token, ak.Function1[HtmlTokenAttribute*, ak.IterationDecision](call = uppercase_values, context = none))
assert(string_is(html_token_attribute(&token, ak.fly_string_from_str("class")) else trap("no class"), "V2"))
# The self-closing flag and its acknowledgement.
assert(not html_token_is_self_closing(&token))
html_token_acknowledge_self_closing_flag_if_set(&token)
assert(not token.m_tag_self_closing_acknowledged)
html_token_set_self_closing(&token, true)
assert(not html_token_has_acknowledged_self_closing_flag(&token))
html_token_acknowledge_self_closing_flag_if_set(&token)
assert(html_token_has_acknowledged_self_closing_flag(&token))
# The tree builder's adjustments.
html_token_adjust_tag_name(&token, ak.fly_string_from_str("b"), ak.fly_string_from_str("x"))
assert(fly_is(html_token_tag_name(&token), "a"))
html_token_adjust_tag_name(&token, ak.fly_string_from_str("a"), ak.fly_string_from_str("textPath"))
assert(fly_is(html_token_tag_name(&token), "textPath"))
html_token_adjust_attribute_name(&token, ak.fly_string_from_str("id"), ak.fly_string_from_str("viewBox"))
assert(html_token_has_attribute(&token, ak.fly_string_from_str("viewBox")) and not html_token_has_attribute(&token, ak.fly_string_from_str("id")))
html_token_adjust_foreign_attribute(&token, ak.fly_string_from_str("href"), ak.fly_string_from_str("xlink"), ak.fly_string_from_str("href"), ak.fly_string_from_str("http://www.w3.org/1999/xlink"))
let href = html_token_raw_attribute(&token, ak.fly_string_from_str("href")) else trap("no href")
assert(fly_is(href.prefix else trap("no prefix"), "xlink"))
assert(fly_is(href.namespace_ else trap("no namespace"), "http://www.w3.org/1999/xlink"))
let class_attribute = html_token_raw_attribute(&token, ak.fly_string_from_str("class")) else trap("no class")
assert(class_attribute.prefix == none and class_attribute.namespace_ == none)
html_token_drop_attributes(&token)
assert(not html_token_has_attributes(&token) and html_token_attribute_count(&token) == 0)
assert(html_token_tag_attributes(&token) == none)
html_token_add_attribute(&token, html_token_attribute_new())
assert(html_token_attribute_count(&token) == 1)
test "HTMLToken::normalize_attributes":
with memory.heap:
let names: str[5] = ["a", "b", "a", "c", "b"]
var token = start_tag_with_attributes("p", names)
assert(not html_token_had_duplicate_attribute(&token))
html_token_normalize_attributes(&token)
assert(html_token_had_duplicate_attribute(&token))
assert(html_token_attribute_count(&token) == 3)
# The first of each name stays.
assert(string_is(html_token_attribute(&token, ak.fly_string_from_str("a")) else trap("no a"), "v0"))
assert(string_is(html_token_attribute(&token, ak.fly_string_from_str("b")) else trap("no b"), "v1"))
assert(fly_is(html_token_last_attribute(&token).local_name, "c"))
test "HTMLToken::to_string":
with memory.heap:
var doctype: HtmlToken
html_token_construct(&doctype, HtmlTokenType.doctype)
html_token_ensure_doctype_data(&doctype).name = ak.string_from_str("html")
html_token_set_start_position(&doctype, HtmlTokenPosition(line = 1, column = 2))
html_token_set_end_position(&doctype, HtmlTokenPosition(line = 3, column = 4))
assert(string_is(html_token_to_string(&doctype), "DOCTYPE { name: 'html' }@1:2-3:4"))
let names: str[2] = ["x", "y"]
var tag = start_tag_with_attributes("div", names)
assert(string_is(html_token_to_string(&tag), "StartTag { name: 'div', { x=\"v0\" y=\"v1\" } }@0:0-0:0"))
var end_tag: HtmlToken
html_token_construct(&end_tag, HtmlTokenType.end_tag)
html_token_set_tag_name(&end_tag, ak.fly_string_from_str("div"))
assert(string_is(html_token_to_string(&end_tag), "EndTag { name: 'div', { } }@0:0-0:0"))
var comment: HtmlToken
html_token_construct(&comment, HtmlTokenType.comment)
html_token_set_comment(&comment, ak.string_from_str(" c "))
assert(string_is(html_token_to_string(&comment), "Comment { data: ' c ' }@0:0-0:0"))
var character = html_token_make_character(0x20AC)
html_token_set_start_position(&character, HtmlTokenPosition(line = 7, column = 8))
assert(string_is(html_token_to_string(&character), "Character { data: '\u{20AC}' }@7:8"))
var eof: HtmlToken
html_token_construct(&eof, HtmlTokenType.end_of_file)
assert(string_is(html_token_to_string(&eof), "EndOfFile@0:0-0:0"))
# mark: The tokenizer's parser hooks ============================================================
# The parser's hook: the context is a bool saying whether the adjusted current node is foreign.
func test_parser_node_is_foreign(parser: void*) -> bool:
return *(bool*)parser
test "HTMLTokenizer: the parser's hooks and switch_to":
with memory.heap:
for foreign in [false, true]:
var is_foreign = foreign
var tokenizer: HtmlTokenizer
html_tokenizer_construct_string_view(&tokenizer, ak.sv("<![CDATA[a]]>"), ak.byte_string_from_str("UTF-8"))
html_tokenizer_set_parser(&tokenizer, test_parser_node_is_foreign, &is_foreign)
var token = html_tokenizer_next_token(&tokenizer) else trap("no token")
if foreign:
# CDATA section state: the characters of the section.
assert(html_token_is_character(&token) and html_token_code_point(&token) == (u32)'a')
else:
# Bogus comment state: "[CDATA[a]]" as a comment.
assert(html_token_is_comment(&token) and string_is(html_token_comment(&token), "[CDATA[a]]"))
# Without a parser, CDATA is a bogus comment (syntax highlighting runs the tokenizer alone).
var tokenizer: HtmlTokenizer
html_tokenizer_construct_string_view(&tokenizer, ak.sv("<![CDATA[a]]><p>x</p>"), ak.byte_string_from_str("UTF-8"))
var comment = html_tokenizer_next_token(&tokenizer) else trap("no token")
assert(html_token_is_comment(&comment))
# The parser switches the tokenizer's state after <p> (as for <plaintext>).
var start_tag = html_tokenizer_next_token(&tokenizer) else trap("no token")
assert(html_token_is_start_tag(&start_tag))
html_tokenizer_switch_to_html_parser(&tokenizer, .plaintext)
for expected in "x</p>".bytes:
var character = html_tokenizer_next_token(&tokenizer) else trap("no token")
assert(html_token_code_point(&character) == expected)
var eof = html_tokenizer_next_token(&tokenizer) else trap("no token")
assert(html_token_is_end_of_file(&eof))
assert(html_tokenizer_next_token(&tokenizer) == none)
test "HTMLTokenizer::state_name":
assert(html_tokenizer_state_name(.data) == "Data")
assert(html_tokenizer_state_name(.rcdata) == "RCDATA")
assert(html_tokenizer_state_name(.before_doctype_name) == "BeforeDOCTYPEName")
assert(html_tokenizer_state_name(.cdata_section_end) == "CDATASectionEnd")
assert(html_tokenizer_state_name(.numeric_character_reference_end) == "NumericCharacterReferenceEnd")