Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-html

Source of release 0.5.1

0.5.1 / src / html_syntax / html_tokenizer_1.lucb

#==============================================================================================
#
#   html_tokenizer_1 - HTMLTokenizer: the input stream and next_token's loop
#
#   DESCRIPTION:
#       Port of Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp (lines 1-319 and 2875-2879):
#       the state machine's macros, the input stream (lines 193-297) and next_token's frame.
#
#       next_token is one function with a label per state in C++, reached by `goto` from the
#       macros. luce-base has no goto (language ยง8.6), so the frame is three nested loops and
#       each state is a function of its own (html_tokenizer_2 to html_tokenizer_8) that answers
#       a TokenizerStep saying which loop to continue:
#
#       - `goto _StartOfFunction` (SWITCH_TO_RETURN_STATE, RECONSUME_IN_RETURN_STATE) continues
#         the outer loop, which hands out a queued token first;
#       - `continue;` and SWITCH_TO continue the `for (;;)` loop, which pauses when the input
#         runs out and otherwise consumes the next input character (SWITCH_TO's own pause check
#         and CONSUME_NEXT_INPUT_CHARACTER are exactly that loop's head);
#       - RECONSUME_IN and EMIT_CHARACTER_AND_RECONSUME_IN (`goto new_state`) continue the
#         innermost loop, which runs m_state again on the same input character;
#       - `return` returns.
#
#       Each macro is a function of the same name that updates the tokenizer as the macro does
#       and answers the step (tokenizer_switch_to for SWITCH_TO, ...). The ON_* conditions are
#       the functions `on`, `on_eof`, `on_ascii_alpha`, ....
#
#==============================================================================================

## AK/Debug.h TOKENIZER_TRACE_DEBUG (0 in the donor build): the dbgln_if messages.
let tokenizer_trace_debug: bool = false

# mark: The state machine's macros ==============================================================

## Where next_token goes after a state has run: the targets of the C++ macros' `goto`,
## `continue` and `return`.
enum TokenizerStep:
    ## `return token;` from next_token.
    returned(token: HtmlToken?)
    ## `continue;` and SWITCH_TO: pause if the input ran out, else consume the next input
    ## character and run m_state.
    consume_next
    ## RECONSUME_IN (`goto new_state;`): run m_state on the same input character.
    reconsume
    ## `goto _StartOfFunction;`: hand out a queued token, else consume the next input character.
    start_of_function

# The value of current_input_character where the C++ reads current_input_character.value().
func input_value(current_input_character: u32?) -> u32:
    return current_input_character else trap("VERIFY: current_input_character.has_value()")

# `m_queued_tokens.enqueue(move(m_current_token))`: the token goes to the queue, and the
# moved-from current token no longer owns its attribute list or DOCTYPE data (its OwnPtr is
# null), so later changes to the current token cannot reach the queued one.
func html_tokenizer_move_current_token(this: HtmlTokenizer*) -> HtmlToken:
    let token = this.m_current_token
    match this.m_current_token.m_data:
        .html_token_doctype_data(_): this.m_current_token.m_data = .html_token_doctype_data(value = none)
        .vector_html_token_attribute(_): this.m_current_token.m_data = .vector_html_token_attribute(value = none)
        _: _ = 0
    return token

# SWITCH_TO(new_state)
func tokenizer_switch_to(this: HtmlTokenizer*, new_state: HtmlTokenizerState) -> TokenizerStep:
    assert(ak.string_builder_is_empty(&this.m_current_builder))
    return tokenizer_switch_to_with_unclean_builder(this, new_state)

# SWITCH_TO_WITH_UNCLEAN_BUILDER(new_state): the pause check and CONSUME_NEXT_INPUT_CHARACTER
# are the head of next_token's `for (;;)` loop.
func tokenizer_switch_to_with_unclean_builder(this: HtmlTokenizer*, new_state: HtmlTokenizerState) -> TokenizerStep:
    html_tokenizer_will_switch_to(this, new_state)
    this.m_state = new_state
    return TokenizerStep.consume_next

# RECONSUME_IN(new_state)
func tokenizer_reconsume_in(this: HtmlTokenizer*, new_state: HtmlTokenizerState) -> TokenizerStep:
    html_tokenizer_will_reconsume_in(this, new_state)
    this.m_state = new_state
    return TokenizerStep.reconsume

# SWITCH_TO_RETURN_STATE
func tokenizer_switch_to_return_state(this: HtmlTokenizer*) -> TokenizerStep:
    html_tokenizer_will_switch_to(this, this.m_return_state)
    this.m_state = this.m_return_state
    return TokenizerStep.start_of_function

# RECONSUME_IN_RETURN_STATE
func tokenizer_reconsume_in_return_state(this: HtmlTokenizer*, current_input_character: u32?) -> TokenizerStep:
    html_tokenizer_will_reconsume_in(this, this.m_return_state)
    this.m_state = this.m_return_state
    if current_input_character != none:
        html_tokenizer_restore_to(this, this.m_prev_offset)
    return TokenizerStep.start_of_function

# SWITCH_TO_AND_EMIT_CURRENT_TOKEN(new_state)
func tokenizer_switch_to_and_emit_current_token(this: HtmlTokenizer*, new_state: HtmlTokenizerState) -> TokenizerStep:
    assert(ak.string_builder_is_empty(&this.m_current_builder))
    html_tokenizer_will_switch_to(this, new_state)
    this.m_state = new_state
    html_tokenizer_will_emit(this, &this.m_current_token)
    ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))
    return .returned(token = ak.queue_dequeue(&this.m_queued_tokens))

# EMIT_CHARACTER_AND_RECONSUME_IN(code_point, new_state)
func tokenizer_emit_character_and_reconsume_in(this: HtmlTokenizer*, code_point: u32, new_state: HtmlTokenizerState) -> TokenizerStep:
    ak.queue_enqueue(&this.m_queued_tokens, html_token_make_character(code_point))
    html_tokenizer_will_reconsume_in(this, new_state)
    this.m_state = new_state
    return TokenizerStep.reconsume

# FLUSH_CODEPOINTS_CONSUMED_AS_A_CHARACTER_REFERENCE
func tokenizer_flush_codepoints_consumed_as_a_character_reference(this: HtmlTokenizer*):
    for code_point in ak.vector_span(&this.m_temporary_buffer):
        if html_tokenizer_consumed_as_part_of_an_attribute(this):
            ak.string_builder_append_code_point(&this.m_current_builder, code_point)
        else:
            html_tokenizer_create_new_token(this, HtmlTokenType.character)
            html_token_set_code_point(&this.m_current_token, code_point)
            ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))

# DONT_CONSUME_NEXT_INPUT_CHARACTER
func tokenizer_dont_consume_next_input_character(this: HtmlTokenizer*, current_input_character: u32?):
    if current_input_character != none:
        html_tokenizer_restore_to(this, this.m_prev_offset)

# ON(code_point): every use in the donor passes an ASCII character or 0.
func on(current_input_character: u32?, code_point: u8) -> bool:
    if let value = current_input_character:
        return value == code_point
    return false

# ON_EOF
func on_eof(current_input_character: u32?) -> bool:
    return current_input_character == none

# ON_ASCII_ALPHA
func on_ascii_alpha(current_input_character: u32?) -> bool:
    if let value = current_input_character:
        return ak.is_ascii_alpha(value)
    return false

# ON_ASCII_ALPHANUMERIC
func on_ascii_alphanumeric(current_input_character: u32?) -> bool:
    if let value = current_input_character:
        return ak.is_ascii_alphanumeric(value)
    return false

# ON_ASCII_UPPER_ALPHA
func on_ascii_upper_alpha(current_input_character: u32?) -> bool:
    if let value = current_input_character:
        return ak.is_ascii_upper_alpha(value)
    return false

# ON_ASCII_LOWER_ALPHA
func on_ascii_lower_alpha(current_input_character: u32?) -> bool:
    if let value = current_input_character:
        return ak.is_ascii_lower_alpha(value)
    return false

# ON_ASCII_DIGIT
func on_ascii_digit(current_input_character: u32?) -> bool:
    if let value = current_input_character:
        return ak.is_ascii_digit(value)
    return false

# ON_ASCII_HEX_DIGIT
func on_ascii_hex_digit(current_input_character: u32?) -> bool:
    if let value = current_input_character:
        return ak.is_ascii_hex_digit(value)
    return false

# ON_WHITESPACE: first_is_one_of(static_cast<char>(c), '\t', '\n', '\f', ' ') of an ASCII c.
func on_whitespace(current_input_character: u32?) -> bool:
    if let value = current_input_character:
        if not ak.is_ascii(value):
            return false
        return value == (u32)'\t' or value == (u32)'\n' or value == 0x0C or value == (u32)' '
    return false

# EMIT_EOF
func tokenizer_emit_eof(this: HtmlTokenizer*) -> TokenizerStep:
    if this.m_has_emitted_eof:
        return .returned(token = none)
    this.m_has_emitted_eof = true
    html_tokenizer_create_new_token(this, HtmlTokenType.end_of_file)
    html_tokenizer_will_emit(this, &this.m_current_token)
    ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))
    return .returned(token = ak.queue_dequeue(&this.m_queued_tokens))

# EMIT_CURRENT_TOKEN_FOLLOWED_BY_EOF
func tokenizer_emit_current_token_followed_by_eof(this: HtmlTokenizer*) -> TokenizerStep:
    assert(ak.string_builder_is_empty(&this.m_current_builder))
    html_tokenizer_will_emit(this, &this.m_current_token)
    ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))

    this.m_has_emitted_eof = true
    html_tokenizer_create_new_token(this, HtmlTokenType.end_of_file)
    html_tokenizer_will_emit(this, &this.m_current_token)
    ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))

    return .returned(token = ak.queue_dequeue(&this.m_queued_tokens))

# EMIT_CHARACTER(code_point)
func tokenizer_emit_character(this: HtmlTokenizer*, code_point: u32) -> TokenizerStep:
    html_tokenizer_create_new_token(this, HtmlTokenType.character)
    html_token_set_code_point(&this.m_current_token, code_point)
    ak.queue_enqueue(&this.m_queued_tokens, html_tokenizer_move_current_token(this))
    return .returned(token = ak.queue_dequeue(&this.m_queued_tokens))

# EMIT_CURRENT_CHARACTER
func tokenizer_emit_current_character(this: HtmlTokenizer*, current_input_character: u32?) -> TokenizerStep:
    return tokenizer_emit_character(this, input_value(current_input_character))

# SWITCH_TO_AND_EMIT_CHARACTER(code_point, new_state)
func tokenizer_switch_to_and_emit_character(this: HtmlTokenizer*, code_point: u32, new_state: HtmlTokenizerState) -> TokenizerStep:
    html_tokenizer_will_switch_to(this, new_state)
    this.m_state = new_state
    return tokenizer_emit_character(this, code_point)

# SWITCH_TO_AND_EMIT_CURRENT_CHARACTER(new_state)
func tokenizer_switch_to_and_emit_current_character(this: HtmlTokenizer*, current_input_character: u32?, new_state: HtmlTokenizerState) -> TokenizerStep:
    return tokenizer_switch_to_and_emit_character(this, input_value(current_input_character), new_state)

# mark: The input stream ========================================================================

## Ported from Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp:193 Web::HTML::log_parse_error.
# The C++ default argument SourceLocation::current() is the caller's luce.location.
func log_parse_error(location: Location = luce.location):
    if tokenizer_trace_debug:
        ak.dbgln(f"Parse error (tokenization) [{location.function} @ {location.file}:{location.line}]")

## Ported from Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp:198 Web::HTML::HTMLTokenizer::should_pause_before_next_input_character.
pub func html_tokenizer_should_pause_before_next_input_character(this: const HtmlTokenizer*, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> bool:
    if stop_at_insertion_point == HtmlTokenizerStopAtInsertionPoint.yes:
        if let insertion_point = this.m_insertion_point:
            if this.m_current_offset >= insertion_point:
                return true

            if this.m_current_offset < (isize)ak.vector_size(&this.m_decoded_input) and *ak.vector_at_const(&this.m_decoded_input, (usize)this.m_current_offset) == (u32)'\r' and this.m_current_offset + 1 == insertion_point:
                # Newline normalization needs one code point of lookahead for CRLF. If document.write()
                # stops the tokenizer immediately after a CR, wait for more input before deciding whether
                # it is a CRLF pair or a standalone CR.
                return true

    if this.m_input_stream_closed:
        return false

    if this.m_current_offset >= (isize)ak.vector_size(&this.m_decoded_input):
        return true

    # Newline normalization needs one code point of lookahead for CRLF. If a network chunk ends in
    # CR, wait for either another code point or the input stream close before deciding whether it is
    # a CRLF pair or a standalone CR.
    return *ak.vector_at_const(&this.m_decoded_input, (usize)this.m_current_offset) == (u32)'\r' and this.m_current_offset + 1 >= (isize)ak.vector_size(&this.m_decoded_input)

## Ported from Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp:226 Web::HTML::HTMLTokenizer::can_run_out_of_characters.
pub func html_tokenizer_can_run_out_of_characters(this: const HtmlTokenizer*, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> bool:
    return (stop_at_insertion_point == HtmlTokenizerStopAtInsertionPoint.yes and this.m_insertion_point != none) or not this.m_input_stream_closed

## Ported from Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp:232 Web::HTML::HTMLTokenizer::next_code_point.
pub func html_tokenizer_next_code_point(this: HtmlTokenizer*, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> u32?:
    if this.m_current_offset >= (isize)ak.vector_size(&this.m_decoded_input):
        return none

    var code_point: u32
    # https://html.spec.whatwg.org/multipage/parsing.html#preprocessing-the-input-stream:tokenization
    # https://infra.spec.whatwg.org/#normalize-newlines
    if (html_tokenizer_peek_code_point(this, 0, stop_at_insertion_point) else 0) == (u32)'\r' and (html_tokenizer_peek_code_point(this, 1, stop_at_insertion_point) else 0) == (u32)'\n':
        # replace every U+000D CR U+000A LF code point pair with a single U+000A LF code point,
        html_tokenizer_skip(this, 2)
        code_point = (u32)'\n'
    elif (html_tokenizer_peek_code_point(this, 0, stop_at_insertion_point) else 0) == (u32)'\r':
        # replace every remaining U+000D CR code point with a U+000A LF code point.
        html_tokenizer_skip(this, 1)
        code_point = (u32)'\n'
    else:
        html_tokenizer_skip(this, 1)
        code_point = *ak.vector_at(&this.m_decoded_input, (usize)this.m_prev_offset)

    if tokenizer_trace_debug:
        ak.dbgln(f"(Tokenizer) Next code_point: {code_point}")
    return code_point

## Ported from Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp:257 Web::HTML::HTMLTokenizer::skip.
pub func html_tokenizer_skip(this: HtmlTokenizer*, count: usize):
    assert(count > 0)
    if not ak.vector_is_empty(&this.m_source_positions):
        ak.vector_append_t(&this.m_source_positions, *ak.vector_last(&this.m_source_positions))
    for i in 0..<count:
        _ = i
        this.m_prev_offset = this.m_current_offset
        let code_point = *ak.vector_at(&this.m_decoded_input, (usize)this.m_current_offset)
        if not ak.vector_is_empty(&this.m_source_positions):
            let last = ak.vector_last(&this.m_source_positions)
            if code_point == (u32)'\n':
                last.column = 0
                last.line += 1
            else:
                last.column += 1
        this.m_current_offset += 1

## Ported from Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp:277 Web::HTML::HTMLTokenizer::peek_code_point.
pub func html_tokenizer_peek_code_point(this: const HtmlTokenizer*, offset: isize, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> u32?:
    let it = this.m_current_offset + offset
    if it >= (isize)ak.vector_size(&this.m_decoded_input):
        return none
    if stop_at_insertion_point == HtmlTokenizerStopAtInsertionPoint.yes:
        if let insertion_point = this.m_insertion_point:
            if it >= insertion_point:
                return none
    return *ak.vector_at_const(&this.m_decoded_input, (usize)it)

## Ported from Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp:290 Web::HTML::HTMLTokenizer::nth_last_position.
pub func html_tokenizer_nth_last_position(this: HtmlTokenizer*, n: usize = 0) -> HtmlTokenPosition:
    if n + 1 > ak.vector_size(&this.m_source_positions):
        if tokenizer_trace_debug:
            ak.dbgln(f"(Tokenizer::nth_last_position) Invalid position requested: {n}th-last of {ak.vector_size(&this.m_source_positions)}. Returning (0-0).")
        return HtmlTokenPosition(line = 0, column = 0)
    return *ak.vector_at(&this.m_source_positions, ak.vector_size(&this.m_source_positions) - 1 - n)

# mark: next_token ==============================================================================

## Ported from Libraries/LibWeb/HTML/Parser/HTMLTokenizer.cpp:299 Web::HTML::HTMLTokenizer::next_token.
pub func html_tokenizer_next_token(this: HtmlTokenizer*, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint = HtmlTokenizerStopAtInsertionPoint.no) -> HtmlToken?:
    if not ak.vector_is_empty(&this.m_source_positions):
        let last_position = *ak.vector_last(&this.m_source_positions)
        ak.vector_clear_with_capacity(&this.m_source_positions)
        ak.vector_append_t(&this.m_source_positions, last_position)
    start_of_function: while true:
        if not ak.queue_is_empty(&this.m_queued_tokens):
            return ak.queue_dequeue(&this.m_queued_tokens)

        if this.m_aborted:
            return none
        consume_next: while true:
            if html_tokenizer_should_pause_before_next_input_character(this, stop_at_insertion_point):
                return none

            let current_input_character = html_tokenizer_next_code_point(this, stop_at_insertion_point)
            reconsume: while true:
                match html_tokenizer_run_state(this, current_input_character, stop_at_insertion_point):
                    .returned(token): return token
                    .consume_next: continue consume_next
                    .reconsume: continue reconsume
                    .start_of_function: continue start_of_function

# `switch (m_state)`: each state's code is the function html_tokenizer_<state>_state.
func html_tokenizer_run_state(this: HtmlTokenizer*, current_input_character: u32?, stop_at_insertion_point: HtmlTokenizerStopAtInsertionPoint) -> TokenizerStep:
    match this.m_state:
        .data: return html_tokenizer_data_state(this, current_input_character, stop_at_insertion_point)
        .rcdata: return html_tokenizer_rcdata_state(this, current_input_character, stop_at_insertion_point)
        .rawtext: return html_tokenizer_rawtext_state(this, current_input_character, stop_at_insertion_point)
        .script_data: return html_tokenizer_script_data_state(this, current_input_character, stop_at_insertion_point)
        .plaintext: return html_tokenizer_plaintext_state(this, current_input_character, stop_at_insertion_point)
        .tag_open: return html_tokenizer_tag_open_state(this, current_input_character, stop_at_insertion_point)
        .end_tag_open: return html_tokenizer_end_tag_open_state(this, current_input_character, stop_at_insertion_point)
        .tag_name: return html_tokenizer_tag_name_state(this, current_input_character, stop_at_insertion_point)
        .rcdata_less_than_sign: return html_tokenizer_rcdata_less_than_sign_state(this, current_input_character, stop_at_insertion_point)
        .rcdata_end_tag_open: return html_tokenizer_rcdata_end_tag_open_state(this, current_input_character, stop_at_insertion_point)
        .rcdata_end_tag_name: return html_tokenizer_rcdata_end_tag_name_state(this, current_input_character, stop_at_insertion_point)
        .rawtext_less_than_sign: return html_tokenizer_rawtext_less_than_sign_state(this, current_input_character, stop_at_insertion_point)
        .rawtext_end_tag_open: return html_tokenizer_rawtext_end_tag_open_state(this, current_input_character, stop_at_insertion_point)
        .rawtext_end_tag_name: return html_tokenizer_rawtext_end_tag_name_state(this, current_input_character, stop_at_insertion_point)
        .script_data_less_than_sign: return html_tokenizer_script_data_less_than_sign_state(this, current_input_character, stop_at_insertion_point)
        .script_data_end_tag_open: return html_tokenizer_script_data_end_tag_open_state(this, current_input_character, stop_at_insertion_point)
        .script_data_end_tag_name: return html_tokenizer_script_data_end_tag_name_state(this, current_input_character, stop_at_insertion_point)
        .script_data_escape_start: return html_tokenizer_script_data_escape_start_state(this, current_input_character, stop_at_insertion_point)
        .script_data_escape_start_dash: return html_tokenizer_script_data_escape_start_dash_state(this, current_input_character, stop_at_insertion_point)
        .script_data_escaped: return html_tokenizer_script_data_escaped_state(this, current_input_character, stop_at_insertion_point)
        .script_data_escaped_dash: return html_tokenizer_script_data_escaped_dash_state(this, current_input_character, stop_at_insertion_point)
        .script_data_escaped_dash_dash: return html_tokenizer_script_data_escaped_dash_dash_state(this, current_input_character, stop_at_insertion_point)
        .script_data_escaped_less_than_sign: return html_tokenizer_script_data_escaped_less_than_sign_state(this, current_input_character, stop_at_insertion_point)
        .script_data_escaped_end_tag_open: return html_tokenizer_script_data_escaped_end_tag_open_state(this, current_input_character, stop_at_insertion_point)
        .script_data_escaped_end_tag_name: return html_tokenizer_script_data_escaped_end_tag_name_state(this, current_input_character, stop_at_insertion_point)
        .script_data_double_escape_start: return html_tokenizer_script_data_double_escape_start_state(this, current_input_character, stop_at_insertion_point)
        .script_data_double_escaped: return html_tokenizer_script_data_double_escaped_state(this, current_input_character, stop_at_insertion_point)
        .script_data_double_escaped_dash: return html_tokenizer_script_data_double_escaped_dash_state(this, current_input_character, stop_at_insertion_point)
        .script_data_double_escaped_dash_dash: return html_tokenizer_script_data_double_escaped_dash_dash_state(this, current_input_character, stop_at_insertion_point)
        .script_data_double_escaped_less_than_sign: return html_tokenizer_script_data_double_escaped_less_than_sign_state(this, current_input_character, stop_at_insertion_point)
        .script_data_double_escape_end: return html_tokenizer_script_data_double_escape_end_state(this, current_input_character, stop_at_insertion_point)
        .before_attribute_name: return html_tokenizer_before_attribute_name_state(this, current_input_character, stop_at_insertion_point)
        .attribute_name: return html_tokenizer_attribute_name_state(this, current_input_character, stop_at_insertion_point)
        .after_attribute_name: return html_tokenizer_after_attribute_name_state(this, current_input_character, stop_at_insertion_point)
        .before_attribute_value: return html_tokenizer_before_attribute_value_state(this, current_input_character, stop_at_insertion_point)
        .attribute_value_double_quoted: return html_tokenizer_attribute_value_double_quoted_state(this, current_input_character, stop_at_insertion_point)
        .attribute_value_single_quoted: return html_tokenizer_attribute_value_single_quoted_state(this, current_input_character, stop_at_insertion_point)
        .attribute_value_unquoted: return html_tokenizer_attribute_value_unquoted_state(this, current_input_character, stop_at_insertion_point)
        .after_attribute_value_quoted: return html_tokenizer_after_attribute_value_quoted_state(this, current_input_character, stop_at_insertion_point)
        .self_closing_start_tag: return html_tokenizer_self_closing_start_tag_state(this, current_input_character, stop_at_insertion_point)
        .bogus_comment: return html_tokenizer_bogus_comment_state(this, current_input_character, stop_at_insertion_point)
        .markup_declaration_open: return html_tokenizer_markup_declaration_open_state(this, current_input_character, stop_at_insertion_point)
        .comment_start: return html_tokenizer_comment_start_state(this, current_input_character, stop_at_insertion_point)
        .comment_start_dash: return html_tokenizer_comment_start_dash_state(this, current_input_character, stop_at_insertion_point)
        .comment: return html_tokenizer_comment_state(this, current_input_character, stop_at_insertion_point)
        .comment_less_than_sign: return html_tokenizer_comment_less_than_sign_state(this, current_input_character, stop_at_insertion_point)
        .comment_less_than_sign_bang: return html_tokenizer_comment_less_than_sign_bang_state(this, current_input_character, stop_at_insertion_point)
        .comment_less_than_sign_bang_dash: return html_tokenizer_comment_less_than_sign_bang_dash_state(this, current_input_character, stop_at_insertion_point)
        .comment_less_than_sign_bang_dash_dash: return html_tokenizer_comment_less_than_sign_bang_dash_dash_state(this, current_input_character, stop_at_insertion_point)
        .comment_end_dash: return html_tokenizer_comment_end_dash_state(this, current_input_character, stop_at_insertion_point)
        .comment_end: return html_tokenizer_comment_end_state(this, current_input_character, stop_at_insertion_point)
        .comment_end_bang: return html_tokenizer_comment_end_bang_state(this, current_input_character, stop_at_insertion_point)
        .doctype: return html_tokenizer_doctype_state(this, current_input_character, stop_at_insertion_point)
        .before_doctype_name: return html_tokenizer_before_doctype_name_state(this, current_input_character, stop_at_insertion_point)
        .doctype_name: return html_tokenizer_doctype_name_state(this, current_input_character, stop_at_insertion_point)
        .after_doctype_name: return html_tokenizer_after_doctype_name_state(this, current_input_character, stop_at_insertion_point)
        .after_doctype_public_keyword: return html_tokenizer_after_doctype_public_keyword_state(this, current_input_character, stop_at_insertion_point)
        .before_doctype_public_identifier: return html_tokenizer_before_doctype_public_identifier_state(this, current_input_character, stop_at_insertion_point)
        .doctype_public_identifier_double_quoted: return html_tokenizer_doctype_public_identifier_double_quoted_state(this, current_input_character, stop_at_insertion_point)
        .doctype_public_identifier_single_quoted: return html_tokenizer_doctype_public_identifier_single_quoted_state(this, current_input_character, stop_at_insertion_point)
        .after_doctype_public_identifier: return html_tokenizer_after_doctype_public_identifier_state(this, current_input_character, stop_at_insertion_point)
        .between_doctype_public_and_system_identifiers: return html_tokenizer_between_doctype_public_and_system_identifiers_state(this, current_input_character, stop_at_insertion_point)
        .after_doctype_system_keyword: return html_tokenizer_after_doctype_system_keyword_state(this, current_input_character, stop_at_insertion_point)
        .before_doctype_system_identifier: return html_tokenizer_before_doctype_system_identifier_state(this, current_input_character, stop_at_insertion_point)
        .doctype_system_identifier_double_quoted: return html_tokenizer_doctype_system_identifier_double_quoted_state(this, current_input_character, stop_at_insertion_point)
        .doctype_system_identifier_single_quoted: return html_tokenizer_doctype_system_identifier_single_quoted_state(this, current_input_character, stop_at_insertion_point)
        .after_doctype_system_identifier: return html_tokenizer_after_doctype_system_identifier_state(this, current_input_character, stop_at_insertion_point)
        .bogus_doctype: return html_tokenizer_bogus_doctype_state(this, current_input_character, stop_at_insertion_point)
        .cdata_section: return html_tokenizer_cdata_section_state(this, current_input_character, stop_at_insertion_point)
        .cdata_section_bracket: return html_tokenizer_cdata_section_bracket_state(this, current_input_character, stop_at_insertion_point)
        .cdata_section_end: return html_tokenizer_cdata_section_end_state(this, current_input_character, stop_at_insertion_point)
        .character_reference: return html_tokenizer_character_reference_state(this, current_input_character, stop_at_insertion_point)
        .named_character_reference: return html_tokenizer_named_character_reference_state(this, current_input_character, stop_at_insertion_point)
        .ambiguous_ampersand: return html_tokenizer_ambiguous_ampersand_state(this, current_input_character, stop_at_insertion_point)
        .numeric_character_reference: return html_tokenizer_numeric_character_reference_state(this, current_input_character, stop_at_insertion_point)
        .hexadecimal_character_reference_start: return html_tokenizer_hexadecimal_character_reference_start_state(this, current_input_character, stop_at_insertion_point)
        .decimal_character_reference_start: return html_tokenizer_decimal_character_reference_start_state(this, current_input_character, stop_at_insertion_point)
        .hexadecimal_character_reference: return html_tokenizer_hexadecimal_character_reference_state(this, current_input_character, stop_at_insertion_point)
        .decimal_character_reference: return html_tokenizer_decimal_character_reference_state(this, current_input_character, stop_at_insertion_point)
        .numeric_character_reference_end: return html_tokenizer_numeric_character_reference_end_state(this, current_input_character, stop_at_insertion_point)
        _: trap("TODO")