Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-html

Source of release 0.5.1

0.5.1 / src / html_syntax / html_tokenizer_2.lucb

#==============================================================================================
#
#   html_tokenizer_2 - HTMLTokenizer states: data, tags, markup declarations, DOCTYPE names
#
#   DESCRIPTION:
#       Port of Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp (lines 318-705): the states
#       Data, TagOpen, TagName, EndTagOpen, MarkupDeclarationOpen, BogusComment, DOCTYPE,
#       BeforeDOCTYPEName, DOCTYPEName and AfterDOCTYPEName of next_token, one function each
#       (see html_tokenizer_1 for how the macros map to TokenizerStep).
#
#==============================================================================================

# mark: Data and tags ===========================================================================

# 13.2.5.1 Data state, https://html.spec.whatwg.org/multipage/parsing.html#data-state
func html_tokenizer_data_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
    if on(current_input_character, '&'):
        this.m_return_state = HtmlTokenizerState.data
        return tokenizer_switch_to(this, .character_reference)
    if on(current_input_character, '<'):
        return tokenizer_switch_to(this, .tag_open)
    if on(current_input_character, 0):
        log_parse_error()
        return tokenizer_emit_current_character(this, current_input_character)
    if on_eof(current_input_character):
        return tokenizer_emit_eof(this)
    return tokenizer_emit_current_character(this, current_input_character)

# 13.2.5.6 Tag open state, https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state
func html_tokenizer_tag_open_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
    if on(current_input_character, '!'):
        return tokenizer_switch_to(this, .markup_declaration_open)
    if on(current_input_character, '/'):
        return tokenizer_switch_to(this, .end_tag_open)
    if on_ascii_alpha(current_input_character):
        html_tokenizer_create_new_token(this, HtmlTokenType.start_tag)
        return tokenizer_reconsume_in(this, .tag_name)
    if on(current_input_character, '?'):
        log_parse_error()
        html_tokenizer_create_new_token(this, HtmlTokenType.comment)
        html_token_set_start_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 2))
        return tokenizer_reconsume_in(this, .bogus_comment)
    if on_eof(current_input_character):
        log_parse_error()
        ak.queue_enqueue(&this.m_queued_tokens, html_token_make_character((u32)'<'))
        return tokenizer_emit_eof(this)
    log_parse_error()
    return tokenizer_emit_character_and_reconsume_in(this, (u32)'<', .data)

# 13.2.5.8 Tag name state, https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state
func html_tokenizer_tag_name_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
    if on_whitespace(current_input_character):
        html_token_set_tag_name(&this.m_current_token, ak.fly_string_from_string(html_tokenizer_consume_current_builder(this)))
        html_token_set_end_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 1))
        return tokenizer_switch_to(this, .before_attribute_name)
    if on(current_input_character, '/'):
        html_token_set_tag_name(&this.m_current_token, ak.fly_string_from_string(html_tokenizer_consume_current_builder(this)))
        html_token_set_end_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 0))
        return tokenizer_switch_to(this, .self_closing_start_tag)
    if on(current_input_character, '>'):
        html_token_set_tag_name(&this.m_current_token, ak.fly_string_from_string(html_tokenizer_consume_current_builder(this)))
        return tokenizer_switch_to_and_emit_current_token(this, .data)
    if on_ascii_upper_alpha(current_input_character):
        ak.string_builder_append_code_point(&this.m_current_builder, ak.to_ascii_lowercase(input_value(current_input_character)))
        html_token_set_end_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 0))
        return TokenizerStep.consume_next
    if on(current_input_character, 0):
        log_parse_error()
        ak.string_builder_append_code_point(&this.m_current_builder, 0xFFFD)
        html_token_set_end_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 0))
        return TokenizerStep.consume_next
    if on_eof(current_input_character):
        log_parse_error()
        return tokenizer_emit_eof(this)
    ak.string_builder_append_code_point(&this.m_current_builder, input_value(current_input_character))
    html_token_set_end_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 0))
    return TokenizerStep.consume_next

# 13.2.5.7 End tag open state, https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state
func html_tokenizer_end_tag_open_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
    if on_ascii_alpha(current_input_character):
        html_tokenizer_create_new_token(this, HtmlTokenType.end_tag)
        return tokenizer_reconsume_in(this, .tag_name)
    if on(current_input_character, '>'):
        log_parse_error()
        return tokenizer_switch_to(this, .data)
    if on_eof(current_input_character):
        log_parse_error()
        ak.queue_enqueue(&this.m_queued_tokens, html_token_make_character((u32)'<'))
        ak.queue_enqueue(&this.m_queued_tokens, html_token_make_character((u32)'/'))
        return tokenizer_emit_eof(this)
    log_parse_error()
    html_tokenizer_create_new_token(this, HtmlTokenType.comment)
    return tokenizer_reconsume_in(this, .bogus_comment)

# mark: Markup declarations and bogus comments ==================================================

# 13.2.5.42 Markup declaration open state, https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state
func html_tokenizer_markup_declaration_open_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
    tokenizer_dont_consume_next_input_character(this, current_input_character)

    match html_tokenizer_consume_next_if_match(this, ak.sv("--"), stop_at_insertion_point):
        .consumed:
            html_tokenizer_create_new_token(this, HtmlTokenType.comment)
            html_token_set_start_position(&this.m_current_token, html_tokenizer_nth_last_position(this, 3))
            return tokenizer_switch_to(this, .comment_start)
        .ran_out_of_characters:
            return .returned(token = none)
        _:
            # ConsumeNextResult::NotConsumed: break;
            _ = 0

    match html_tokenizer_consume_next_if_match(this, ak.sv("DOCTYPE"), stop_at_insertion_point, ak.CaseSensitivity.case_insensitive):
        .consumed:
            return tokenizer_switch_to(this, .doctype)
        .ran_out_of_characters:
            return .returned(token = none)
        _:
            # ConsumeNextResult::NotConsumed: break;
            _ = 0

    match html_tokenizer_consume_next_if_match(this, ak.sv("[CDATA["), stop_at_insertion_point):
        .consumed:
            # We keep the parser optional so that syntax highlighting can be lexer-only.
            # The parser registers itself with the lexer it creates.
            # (m_parser != nullptr && m_parser->adjusted_current_node() &&
            # m_parser->adjusted_current_node()->namespace_uri() != Namespace::HTML) is the
            # parser's hook, DESIGN.md ยง4.1.3.
            if html_tokenizer_adjusted_current_node_is_foreign(this):
                return tokenizer_switch_to(this, .cdata_section)
            html_tokenizer_create_new_token(this, HtmlTokenType.comment)
            ak.string_builder_append(&this.m_current_builder, "[CDATA[")
            return tokenizer_switch_to_with_unclean_builder(this, .bogus_comment)
        .ran_out_of_characters:
            return .returned(token = none)
        _:
            # ConsumeNextResult::NotConsumed: break;
            _ = 0

    log_parse_error()
    html_tokenizer_create_new_token(this, HtmlTokenType.comment)
    return tokenizer_switch_to(this, .bogus_comment)

# 13.2.5.41 Bogus comment state, https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state
func html_tokenizer_bogus_comment_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
    if on(current_input_character, '>'):
        html_token_set_comment(&this.m_current_token, html_tokenizer_consume_current_builder(this))
        return tokenizer_switch_to_and_emit_current_token(this, .data)
    if on_eof(current_input_character):
        html_token_set_comment(&this.m_current_token, html_tokenizer_consume_current_builder(this))
        return tokenizer_emit_current_token_followed_by_eof(this)
    if on(current_input_character, 0):
        log_parse_error()
        ak.string_builder_append_code_point(&this.m_current_builder, 0xFFFD)
        return TokenizerStep.consume_next
    ak.string_builder_append_code_point(&this.m_current_builder, input_value(current_input_character))
    return TokenizerStep.consume_next

# mark: DOCTYPE and its name ====================================================================

# 13.2.5.53 DOCTYPE state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-state
func html_tokenizer_doctype_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
    if on_whitespace(current_input_character):
        return tokenizer_switch_to(this, .before_doctype_name)
    if on(current_input_character, '>'):
        return tokenizer_reconsume_in(this, .before_doctype_name)
    if on_eof(current_input_character):
        log_parse_error()
        html_tokenizer_create_new_token(this, HtmlTokenType.doctype)
        html_token_ensure_doctype_data(&this.m_current_token).force_quirks = true
        ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))
        return tokenizer_emit_eof(this)
    log_parse_error()
    return tokenizer_reconsume_in(this, .before_doctype_name)

# 13.2.5.54 Before DOCTYPE name state, https://html.spec.whatwg.org/multipage/parsing.html#before-doctype-name-state
func html_tokenizer_before_doctype_name_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
    if on_whitespace(current_input_character):
        return TokenizerStep.consume_next
    if on_ascii_upper_alpha(current_input_character):
        html_tokenizer_create_new_token(this, HtmlTokenType.doctype)
        ak.string_builder_append_code_point(&this.m_current_builder, ak.to_ascii_lowercase(input_value(current_input_character)))
        html_token_ensure_doctype_data(&this.m_current_token).missing_name = false
        return tokenizer_switch_to_with_unclean_builder(this, .doctype_name)
    if on(current_input_character, 0):
        log_parse_error()
        html_tokenizer_create_new_token(this, HtmlTokenType.doctype)
        ak.string_builder_append_code_point(&this.m_current_builder, 0xFFFD)
        html_token_ensure_doctype_data(&this.m_current_token).missing_name = false
        return tokenizer_switch_to_with_unclean_builder(this, .doctype_name)
    if on(current_input_character, '>'):
        log_parse_error()
        html_tokenizer_create_new_token(this, HtmlTokenType.doctype)
        html_token_ensure_doctype_data(&this.m_current_token).force_quirks = true
        return tokenizer_switch_to_and_emit_current_token(this, .data)
    if on_eof(current_input_character):
        log_parse_error()
        html_tokenizer_create_new_token(this, HtmlTokenType.doctype)
        html_token_ensure_doctype_data(&this.m_current_token).force_quirks = true
        ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))
        return tokenizer_emit_eof(this)
    html_tokenizer_create_new_token(this, HtmlTokenType.doctype)
    ak.string_builder_append_code_point(&this.m_current_builder, input_value(current_input_character))
    html_token_ensure_doctype_data(&this.m_current_token).missing_name = false
    return tokenizer_switch_to_with_unclean_builder(this, .doctype_name)

# 13.2.5.55 DOCTYPE name state, https://html.spec.whatwg.org/multipage/parsing.html#doctype-name-state
func html_tokenizer_doctype_name_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
    if on_whitespace(current_input_character):
        html_token_ensure_doctype_data(&this.m_current_token).name = html_tokenizer_consume_current_builder(this)
        return tokenizer_switch_to(this, .after_doctype_name)
    if on(current_input_character, '>'):
        html_token_ensure_doctype_data(&this.m_current_token).name = html_tokenizer_consume_current_builder(this)
        return tokenizer_switch_to_and_emit_current_token(this, .data)
    if on_ascii_upper_alpha(current_input_character):
        ak.string_builder_append_code_point(&this.m_current_builder, ak.to_ascii_lowercase(input_value(current_input_character)))
        return TokenizerStep.consume_next
    if on(current_input_character, 0):
        log_parse_error()
        ak.string_builder_append_code_point(&this.m_current_builder, 0xFFFD)
        return TokenizerStep.consume_next
    if on_eof(current_input_character):
        log_parse_error()
        html_token_ensure_doctype_data(&this.m_current_token).force_quirks = true
        ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))
        return tokenizer_emit_eof(this)
    ak.string_builder_append_code_point(&this.m_current_builder, input_value(current_input_character))
    return TokenizerStep.consume_next

# 13.2.5.56 After DOCTYPE name state, https://html.spec.whatwg.org/multipage/parsing.html#after-doctype-name-state
func html_tokenizer_after_doctype_name_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
    if on_whitespace(current_input_character):
        return TokenizerStep.consume_next
    if on(current_input_character, '>'):
        return tokenizer_switch_to_and_emit_current_token(this, .data)
    if on_eof(current_input_character):
        log_parse_error()
        html_token_ensure_doctype_data(&this.m_current_token).force_quirks = true
        ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))
        return tokenizer_emit_eof(this)
    if ak.to_ascii_uppercase(input_value(current_input_character)) == (u32)'P':
        match html_tokenizer_consume_next_if_match(this, ak.sv("UBLIC"), stop_at_insertion_point, ak.CaseSensitivity.case_insensitive):
            .consumed:
                return tokenizer_switch_to(this, .after_doctype_public_keyword)
            .ran_out_of_characters:
                tokenizer_dont_consume_next_input_character(this, current_input_character)
                return .returned(token = none)
            _:
                # ConsumeNextResult::NotConsumed: break;
                _ = 0
    if ak.to_ascii_uppercase(input_value(current_input_character)) == (u32)'S':
        match html_tokenizer_consume_next_if_match(this, ak.sv("YSTEM"), stop_at_insertion_point, ak.CaseSensitivity.case_insensitive):
            .consumed:
                return tokenizer_switch_to(this, .after_doctype_system_keyword)
            .ran_out_of_characters:
                tokenizer_dont_consume_next_input_character(this, current_input_character)
                return .returned(token = none)
            _:
                # ConsumeNextResult::NotConsumed: break;
                _ = 0
    log_parse_error()
    html_token_ensure_doctype_data(&this.m_current_token).force_quirks = true
    return tokenizer_reconsume_in(this, .bogus_doctype)