dymokomi/luce-browser-html
Source of release 0.5.1
0.5.1 / src / html_syntax / html_tokenizer_2.lucb
#==============================================================================================
#
# html_tokenizer_2 - HTMLTokenizer states: data, tags, markup declarations, DOCTYPE names
#
# DESCRIPTION:
# Port of Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp (lines 318-705): the states
# Data, TagOpen, TagName, EndTagOpen, MarkupDeclarationOpen, BogusComment, DOCTYPE,
# BeforeDOCTYPEName, DOCTYPEName and AfterDOCTYPEName of next_token, one function each
# (see html_tokenizer_1 for how the macros map to TokenizerStep).
#
#==============================================================================================
# mark: Data and tags ===========================================================================
# 13.2.5.1 Data state, https://html.spec.whatwg.org/multipage/parsing.html#data-state
func html_tokenizer_data_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
if on(current_input_character, '&'):
this.m_return_state = HtmlTokenizerState.data
return tokenizer_switch_to(this, .character_reference)
if on(current_input_character, '<'):
return tokenizer_switch_to(this, .tag_open)
if on(current_input_character, 0):
log_parse_error()
return tokenizer_emit_current_character(this, current_input_character)
if on_eof(current_input_character):
return tokenizer_emit_eof(this)
return tokenizer_emit_current_character(this, current_input_character)
# 13.2.5.6 Tag open state, https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
func html_tokenizer_tag_open_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
if on(current_input_character, '!'):
return tokenizer_switch_to(this, .markup_declaration_open)
if on(current_input_character, '/'):
return tokenizer_switch_to(this, .end_tag_open)
if on_ascii_alpha(current_input_character):
html_tokenizer_create_new_token(this, HtmlTokenType.start_tag)
return tokenizer_reconsume_in(this, .tag_name)
if on(current_input_character, '?'):
log_parse_error()
html_tokenizer_create_new_token(this, HtmlTokenType.comment)
html_token_set_start_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 2))
return tokenizer_reconsume_in(this, .bogus_comment)
if on_eof(current_input_character):
log_parse_error()
ak.queue_enqueue(&this.m_queued_tokens, html_token_make_character((u32)'<'))
return tokenizer_emit_eof(this)
log_parse_error()
return tokenizer_emit_character_and_reconsume_in(this, (u32)'<', .data)
# 13.2.5.8 Tag name state, https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
func html_tokenizer_tag_name_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
if on_whitespace(current_input_character):
html_token_set_tag_name(&this.m_current_token, ak.fly_string_from_string(html_tokenizer_consume_current_builder(this)))
html_token_set_end_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 1))
return tokenizer_switch_to(this, .before_attribute_name)
if on(current_input_character, '/'):
html_token_set_tag_name(&this.m_current_token, ak.fly_string_from_string(html_tokenizer_consume_current_builder(this)))
html_token_set_end_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 0))
return tokenizer_switch_to(this, .self_closing_start_tag)
if on(current_input_character, '>'):
html_token_set_tag_name(&this.m_current_token, ak.fly_string_from_string(html_tokenizer_consume_current_builder(this)))
return tokenizer_switch_to_and_emit_current_token(this, .data)
if on_ascii_upper_alpha(current_input_character):
ak.string_builder_append_code_point(&this.m_current_builder, ak.to_ascii_lowercase(input_value(current_input_character)))
html_token_set_end_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 0))
return TokenizerStep.consume_next
if on(current_input_character, 0):
log_parse_error()
ak.string_builder_append_code_point(&this.m_current_builder, 0xFFFD)
html_token_set_end_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 0))
return TokenizerStep.consume_next
if on_eof(current_input_character):
log_parse_error()
return tokenizer_emit_eof(this)
ak.string_builder_append_code_point(&this.m_current_builder, input_value(current_input_character))
html_token_set_end_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 0))
return TokenizerStep.consume_next
# 13.2.5.7 End tag open state, https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state
func html_tokenizer_end_tag_open_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
if on_ascii_alpha(current_input_character):
html_tokenizer_create_new_token(this, HtmlTokenType.end_tag)
return tokenizer_reconsume_in(this, .tag_name)
if on(current_input_character, '>'):
log_parse_error()
return tokenizer_switch_to(this, .data)
if on_eof(current_input_character):
log_parse_error()
ak.queue_enqueue(&this.m_queued_tokens, html_token_make_character((u32)'<'))
ak.queue_enqueue(&this.m_queued_tokens, html_token_make_character((u32)'/'))
return tokenizer_emit_eof(this)
log_parse_error()
html_tokenizer_create_new_token(this, HtmlTokenType.comment)
return tokenizer_reconsume_in(this, .bogus_comment)
# mark: Markup declarations and bogus comments ==================================================
# 13.2.5.42 Markup declaration open state, https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state
func html_tokenizer_markup_declaration_open_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
tokenizer_dont_consume_next_input_character(this, current_input_character)
match html_tokenizer_consume_next_if_match(this, ak.sv("--"), stop_at_insertion_point):
.consumed:
html_tokenizer_create_new_token(this, HtmlTokenType.comment)
html_token_set_start_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 3))
return tokenizer_switch_to(this, .comment_start)
.ran_out_of_characters:
return .returned(token = none)
_:
# ConsumeNextResult::NotConsumed: break;
_ = 0
match html_tokenizer_consume_next_if_match(this, ak.sv("DOCTYPE"), stop_at_insertion_point, ak.CaseSensitivity.case_insensitive):
.consumed:
return tokenizer_switch_to(this, .doctype)
.ran_out_of_characters:
return .returned(token = none)
_:
# ConsumeNextResult::NotConsumed: break;
_ = 0
match html_tokenizer_consume_next_if_match(this, ak.sv("[CDATA["), stop_at_insertion_point):
.consumed:
# We keep the parser optional so that syntax highlighting can be lexer-only.
# The parser registers itself with the lexer it creates.
# (m_parser != nullptr && m_parser->adjusted_current_node() &&
# m_parser->adjusted_current_node()->namespace_uri() != Namespace::HTML) is the
# parser's hook, DESIGN.md ยง4.1.3.
if html_tokenizer_adjusted_current_node_is_foreign(this):
return tokenizer_switch_to(this, .cdata_section)
html_tokenizer_create_new_token(this, HtmlTokenType.comment)
ak.string_builder_append(&this.m_current_builder, "[CDATA[")
return tokenizer_switch_to_with_unclean_builder(this, .bogus_comment)
.ran_out_of_characters:
return .returned(token = none)
_:
# ConsumeNextResult::NotConsumed: break;
_ = 0
log_parse_error()
html_tokenizer_create_new_token(this, HtmlTokenType.comment)
return tokenizer_switch_to(this, .bogus_comment)
# 13.2.5.41 Bogus comment state, https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state
func html_tokenizer_bogus_comment_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
if on(current_input_character, '>'):
html_token_set_comment(&this.m_current_token, html_tokenizer_consume_current_builder(this))
return tokenizer_switch_to_and_emit_current_token(this, .data)
if on_eof(current_input_character):
html_token_set_comment(&this.m_current_token, html_tokenizer_consume_current_builder(this))
return tokenizer_emit_current_token_followed_by_eof(this)
if on(current_input_character, 0):
log_parse_error()
ak.string_builder_append_code_point(&this.m_current_builder, 0xFFFD)
return TokenizerStep.consume_next
ak.string_builder_append_code_point(&this.m_current_builder, input_value(current_input_character))
return TokenizerStep.consume_next
# mark: DOCTYPE and its name ====================================================================
# 13.2.5.53 DOCTYPE state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-state
func html_tokenizer_doctype_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
if on_whitespace(current_input_character):
return tokenizer_switch_to(this, .before_doctype_name)
if on(current_input_character, '>'):
return tokenizer_reconsume_in(this, .before_doctype_name)
if on_eof(current_input_character):
log_parse_error()
html_tokenizer_create_new_token(this, HtmlTokenType.doctype)
html_token_ensure_doctype_data(&this.m_current_token).force_quirks = true
ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))
return tokenizer_emit_eof(this)
log_parse_error()
return tokenizer_reconsume_in(this, .before_doctype_name)
# 13.2.5.54 Before DOCTYPE name state, https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-name-state
func html_tokenizer_before_doctype_name_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
if on_whitespace(current_input_character):
return TokenizerStep.consume_next
if on_ascii_upper_alpha(current_input_character):
html_tokenizer_create_new_token(this, HtmlTokenType.doctype)
ak.string_builder_append_code_point(&this.m_current_builder, ak.to_ascii_lowercase(input_value(current_input_character)))
html_token_ensure_doctype_data(&this.m_current_token).missing_name = false
return tokenizer_switch_to_with_unclean_builder(this, .doctype_name)
if on(current_input_character, 0):
log_parse_error()
html_tokenizer_create_new_token(this, HtmlTokenType.doctype)
ak.string_builder_append_code_point(&this.m_current_builder, 0xFFFD)
html_token_ensure_doctype_data(&this.m_current_token).missing_name = false
return tokenizer_switch_to_with_unclean_builder(this, .doctype_name)
if on(current_input_character, '>'):
log_parse_error()
html_tokenizer_create_new_token(this, HtmlTokenType.doctype)
html_token_ensure_doctype_data(&this.m_current_token).force_quirks = true
return tokenizer_switch_to_and_emit_current_token(this, .data)
if on_eof(current_input_character):
log_parse_error()
html_tokenizer_create_new_token(this, HtmlTokenType.doctype)
html_token_ensure_doctype_data(&this.m_current_token).force_quirks = true
ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))
return tokenizer_emit_eof(this)
html_tokenizer_create_new_token(this, HtmlTokenType.doctype)
ak.string_builder_append_code_point(&this.m_current_builder, input_value(current_input_character))
html_token_ensure_doctype_data(&this.m_current_token).missing_name = false
return tokenizer_switch_to_with_unclean_builder(this, .doctype_name)
# 13.2.5.55 DOCTYPE name state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-name-state
func html_tokenizer_doctype_name_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
if on_whitespace(current_input_character):
html_token_ensure_doctype_data(&this.m_current_token).name = html_tokenizer_consume_current_builder(this)
return tokenizer_switch_to(this, .after_doctype_name)
if on(current_input_character, '>'):
html_token_ensure_doctype_data(&this.m_current_token).name = html_tokenizer_consume_current_builder(this)
return tokenizer_switch_to_and_emit_current_token(this, .data)
if on_ascii_upper_alpha(current_input_character):
ak.string_builder_append_code_point(&this.m_current_builder, ak.to_ascii_lowercase(input_value(current_input_character)))
return TokenizerStep.consume_next
if on(current_input_character, 0):
log_parse_error()
ak.string_builder_append_code_point(&this.m_current_builder, 0xFFFD)
return TokenizerStep.consume_next
if on_eof(current_input_character):
log_parse_error()
html_token_ensure_doctype_data(&this.m_current_token).force_quirks = true
ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))
return tokenizer_emit_eof(this)
ak.string_builder_append_code_point(&this.m_current_builder, input_value(current_input_character))
return TokenizerStep.consume_next
# 13.2.5.56 After DOCTYPE name state, https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-name-state
func html_tokenizer_after_doctype_name_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
if on_whitespace(current_input_character):
return TokenizerStep.consume_next
if on(current_input_character, '>'):
return tokenizer_switch_to_and_emit_current_token(this, .data)
if on_eof(current_input_character):
log_parse_error()
html_token_ensure_doctype_data(&this.m_current_token).force_quirks = true
ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))
return tokenizer_emit_eof(this)
if ak.to_ascii_uppercase(input_value(current_input_character)) == (u32)'P':
match html_tokenizer_consume_next_if_match(this, ak.sv("UBLIC"), stop_at_insertion_point, ak.CaseSensitivity.case_insensitive):
.consumed:
return tokenizer_switch_to(this, .after_doctype_public_keyword)
.ran_out_of_characters:
tokenizer_dont_consume_next_input_character(this, current_input_character)
return .returned(token = none)
_:
# ConsumeNextResult::NotConsumed: break;
_ = 0
if ak.to_ascii_uppercase(input_value(current_input_character)) == (u32)'S':
match html_tokenizer_consume_next_if_match(this, ak.sv("YSTEM"), stop_at_insertion_point, ak.CaseSensitivity.case_insensitive):
.consumed:
return tokenizer_switch_to(this, .after_doctype_system_keyword)
.ran_out_of_characters:
tokenizer_dont_consume_next_input_character(this, current_input_character)
return .returned(token = none)
_:
# ConsumeNextResult::NotConsumed: break;
_ = 0
log_parse_error()
html_token_ensure_doctype_data(&this.m_current_token).force_quirks = true
return tokenizer_reconsume_in(this, .bogus_doctype)