Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-css

Source of release 0.5.1

0.5.1 / src / css_syntax / tokenizer_3.lucb

#==============================================================================================
#
#   tokenizer_3 - CSS::Parser::Tokenizer: consume a token, and the input's byte offsets
#
#   DESCRIPTION:
#       Port of LibWeb/CSS/Parser/Tokenizer.cpp (lines 964-1242: consume_a_token,
#       current_byte_offset, input_since).
#
#==============================================================================================

# mark: Consume a token =========================================================================

## Ported from Libraries/LibWeb/CSS/Parser/Tokenizer.cpp:965 Web::CSS::Parser::Tokenizer::consume_a_token.
# https://www.w3.org/TR/css-syntax-3/#consume-token
pub func tokenizer_consume_a_token(this: Tokenizer*) -> Token:
    # This section describes how to consume a token from a stream of code points.
    # It will return a single token of any type.

    let start_byte_offset = tokenizer_current_byte_offset(this)

    # Consume comments.
    tokenizer_consume_comments(this)

    # AD-HOC: Preserve comments as whitespace tokens, for serializing custom properties.
    let after_comments_byte_offset = tokenizer_current_byte_offset(this)
    if after_comments_byte_offset != start_byte_offset:
        return token_create_whitespace(tokenizer_input_since(this, start_byte_offset))

    # Consume the next input code point.
    let input = tokenizer_next_code_point(this)

    # whitespace
    if is_whitespace(input):
        if css_tokenizer_debug:
            ak.dbgln("is whitespace")
        # Consume as much whitespace as possible. Return a <whitespace-token>.
        tokenizer_consume_as_much_whitespace_as_possible(this)
        return token_create_whitespace(tokenizer_input_since(this, start_byte_offset))

    # U+0022 QUOTATION MARK (")
    if is_quotation_mark(input):
        if css_tokenizer_debug:
            ak.dbgln("is quotation mark")
        # Consume a string token and return it.
        return tokenizer_consume_string_token(this, input)

    # U+0023 NUMBER SIGN (#)
    if is_number_sign(input):
        if css_tokenizer_debug:
            ak.dbgln("is number sign")

        # If the next input code point is an ident code point or the next two input code points
        # are a valid escape, then:
        let next_input = tokenizer_peek_code_point(this)
        let maybe_escape = tokenizer_peek_twin(this)

        if is_ident_code_point(next_input) or tokenizer_is_valid_escape_sequence(maybe_escape):
            # 1. Create a <hash-token>.

            # 2. If the next 3 input code points would start an ident sequence, set the <hash-token>’s
            #    type flag to "id".
            var hash_type = TokenHashType.unrestricted
            if tokenizer_would_start_an_ident_sequence(tokenizer_peek_triplet(this)):
                hash_type = TokenHashType.id

            # 3. Consume an ident sequence, and set the <hash-token>’s value to the returned string.
            let value = tokenizer_consume_an_ident_sequence(this)

            # 4. Return the <hash-token>.
            return token_create_hash(value, hash_type, tokenizer_input_since(this, start_byte_offset))

        # Otherwise, return a <delim-token> with its value set to the current input code point.
        return token_create_delim(input, tokenizer_input_since(this, start_byte_offset))

    # U+0027 APOSTROPHE (')
    if is_apostrophe(input):
        if css_tokenizer_debug:
            ak.dbgln("is apostrophe")
        # Consume a string token and return it.
        return tokenizer_consume_string_token(this, input)

    # U+0028 LEFT PARENTHESIS (()
    if is_left_paren(input):
        if css_tokenizer_debug:
            ak.dbgln("is left paren")
        # Return a <(-token>.
        return token_create(TokenType.open_paren, tokenizer_input_since(this, start_byte_offset))

    # U+0029 RIGHT PARENTHESIS ())
    if is_right_paren(input):
        if css_tokenizer_debug:
            ak.dbgln("is right paren")
        # Return a <)-token>.
        return token_create(TokenType.close_paren, tokenizer_input_since(this, start_byte_offset))

    # U+002B PLUS SIGN (+)
    if is_plus_sign(input):
        if css_tokenizer_debug:
            ak.dbgln("is plus sign")
        # If the input stream starts with a number, reconsume the current input code point,
        # consume a numeric token and return it.
        if tokenizer_would_start_a_number(tokenizer_start_of_input_stream_triplet(this)):
            tokenizer_reconsume_current_input_code_point(this)
            return tokenizer_consume_a_numeric_token(this)

        # Otherwise, return a <delim-token> with its value set to the current input code point.
        return token_create_delim(input, tokenizer_input_since(this, start_byte_offset))

    # U+002C COMMA (,)
    if is_comma(input):
        if css_tokenizer_debug:
            ak.dbgln("is comma")
        # Return a <comma-token>.
        return token_create(TokenType.comma, tokenizer_input_since(this, start_byte_offset))

    # U+002D HYPHEN-MINUS (-)
    if is_hyphen_minus(input):
        if css_tokenizer_debug:
            ak.dbgln("is hyphen minus")
        # If the input stream starts with a number, reconsume the current input code point,
        # consume a numeric token, and return it.
        if tokenizer_would_start_a_number(tokenizer_start_of_input_stream_triplet(this)):
            tokenizer_reconsume_current_input_code_point(this)
            return tokenizer_consume_a_numeric_token(this)

        # Otherwise, if the next 2 input code points are U+002D HYPHEN-MINUS U+003E
        # GREATER-THAN SIGN (->), consume them and return a <CDC-token>.
        let next_twin = tokenizer_peek_twin(this)
        if is_hyphen_minus(next_twin.first) and is_greater_than_sign(next_twin.second):
            _ = tokenizer_next_code_point(this)
            _ = tokenizer_next_code_point(this)

            return token_create(TokenType.cdc, tokenizer_input_since(this, start_byte_offset))

        # Otherwise, if the input stream starts with an identifier, reconsume the current
        # input code point, consume an ident-like token, and return it.
        if tokenizer_would_start_an_ident_sequence(tokenizer_start_of_input_stream_triplet(this)):
            tokenizer_reconsume_current_input_code_point(this)
            return tokenizer_consume_an_ident_like_token(this)

        # Otherwise, return a <delim-token> with its value set to the current input code point.
        return token_create_delim(input, tokenizer_input_since(this, start_byte_offset))

    # U+002E FULL STOP (.)
    if is_full_stop(input):
        if css_tokenizer_debug:
            ak.dbgln("is full stop")
        # If the input stream starts with a number, reconsume the current input code point,
        # consume a numeric token, and return it.
        if tokenizer_would_start_a_number(tokenizer_start_of_input_stream_triplet(this)):
            tokenizer_reconsume_current_input_code_point(this)
            return tokenizer_consume_a_numeric_token(this)

        # Otherwise, return a <delim-token> with its value set to the current input code point.
        return token_create_delim(input, tokenizer_input_since(this, start_byte_offset))

    # U+003A COLON (:)
    if is_colon(input):
        if css_tokenizer_debug:
            ak.dbgln("is colon")
        # Return a <colon-token>.
        return token_create(TokenType.colon, tokenizer_input_since(this, start_byte_offset))

    # U+003B SEMICOLON (;)
    if is_semicolon(input):
        if css_tokenizer_debug:
            ak.dbgln("is semicolon")
        # Return a <semicolon-token>.
        return token_create(TokenType.semicolon, tokenizer_input_since(this, start_byte_offset))

    # U+003C LESS-THAN SIGN (<)
    if is_less_than_sign(input):
        if css_tokenizer_debug:
            ak.dbgln("is less than")
        # If the next 3 input code points are U+0021 EXCLAMATION MARK U+002D HYPHEN-MINUS
        # U+002D HYPHEN-MINUS (!--), consume them and return a <CDO-token>.
        let maybe_cdo = tokenizer_peek_triplet(this)
        if is_exclamation_mark(maybe_cdo.first) and is_hyphen_minus(maybe_cdo.second) and is_hyphen_minus(maybe_cdo.third):
            _ = tokenizer_next_code_point(this)
            _ = tokenizer_next_code_point(this)
            _ = tokenizer_next_code_point(this)

            return token_create(TokenType.cdo, tokenizer_input_since(this, start_byte_offset))

        # Otherwise, return a <delim-token> with its value set to the current input code point.
        return token_create_delim(input, tokenizer_input_since(this, start_byte_offset))

    # U+0040 COMMERCIAL AT (@)
    if is_at(input):
        if css_tokenizer_debug:
            ak.dbgln("is at")
        # If the next 3 input code points would start an ident sequence, consume an ident sequence, create
        # an <at-keyword-token> with its value set to the returned value, and return it.
        if tokenizer_would_start_an_ident_sequence(tokenizer_peek_triplet(this)):
            let name = tokenizer_consume_an_ident_sequence(this)
            return token_create_at_keyword(name, tokenizer_input_since(this, start_byte_offset))

        # Otherwise, return a <delim-token> with its value set to the current input code point.
        return token_create_delim(input, tokenizer_input_since(this, start_byte_offset))

    # U+005B LEFT SQUARE BRACKET ([)
    if is_open_square_bracket(input):
        if css_tokenizer_debug:
            ak.dbgln("is open square")
        # Return a <[-token>.
        return token_create(TokenType.open_square, tokenizer_input_since(this, start_byte_offset))

    # U+005C REVERSE SOLIDUS (\)
    if is_reverse_solidus(input):
        if css_tokenizer_debug:
            ak.dbgln("is reverse solidus")
        # If the input stream starts with a valid escape, reconsume the current input code point,
        # consume an ident-like token, and return it.
        if tokenizer_is_valid_escape_sequence(tokenizer_start_of_input_stream_twin(this)):
            tokenizer_reconsume_current_input_code_point(this)
            return tokenizer_consume_an_ident_like_token(this)

        # Otherwise, this is a parse error. Return a <delim-token> with its value set to the
        # current input code point.
        log_parse_error()
        return token_create_delim(input, tokenizer_input_since(this, start_byte_offset))

    # U+005D RIGHT SQUARE BRACKET (])
    if is_closed_square_bracket(input):
        if css_tokenizer_debug:
            ak.dbgln("is closed square")
        # Return a <]-token>.
        return token_create(TokenType.close_square, tokenizer_input_since(this, start_byte_offset))

    # U+007B LEFT CURLY BRACKET ({)
    if is_open_curly_bracket(input):
        if css_tokenizer_debug:
            ak.dbgln("is open curly")
        # Return a <{-token>.
        return token_create(TokenType.open_curly, tokenizer_input_since(this, start_byte_offset))

    # U+007D RIGHT CURLY BRACKET (})
    if is_closed_curly_bracket(input):
        if css_tokenizer_debug:
            ak.dbgln("is closed curly")
        # Return a <}-token>.
        return token_create(TokenType.close_curly, tokenizer_input_since(this, start_byte_offset))

    # digit
    if is_digit(input):
        if css_tokenizer_debug:
            ak.dbgln("is digit")
        # Reconsume the current input code point, consume a numeric token, and return it.
        tokenizer_reconsume_current_input_code_point(this)
        return tokenizer_consume_a_numeric_token(this)

    # name-start code point
    if is_ident_start_code_point(input):
        if css_tokenizer_debug:
            ak.dbgln("is name start")
        # Reconsume the current input code point, consume an ident-like token, and return it.
        tokenizer_reconsume_current_input_code_point(this)
        return tokenizer_consume_an_ident_like_token(this)

    # EOF
    if is_eof(input):
        # Return an <EOF-token>.
        return tokenizer_create_eof_token()

    # anything else
    if css_tokenizer_debug:
        ak.dbgln("is delimiter")
    # Return a <delim-token> with its value set to the current input code point.
    return token_create_delim(input, tokenizer_input_since(this, start_byte_offset))

# mark: Byte offsets ============================================================================

## Ported from Libraries/LibWeb/CSS/Parser/Tokenizer.cpp:1232 Web::CSS::Parser::Tokenizer::current_byte_offset.
pub func tokenizer_current_byte_offset(this: const Tokenizer*) -> usize:
    # m_utf8_iterator.ptr() - m_utf8_view.bytes(); both are null only for an empty input.
    let current = ak.utf8_code_point_iterator_ptr(&this.m_utf8_iterator) else return 0
    let start = ak.utf8_view_bytes(&this.m_utf8_view) else return 0
    return (usize)current - (usize)start

## Ported from Libraries/LibWeb/CSS/Parser/Tokenizer.cpp:1237 Web::CSS::Parser::Tokenizer::input_since.
pub func tokenizer_input_since(this: const Tokenizer*, offset: usize) -> ak.String:
    return ak.string_substring_from_byte_offset_with_shared_superstring_usize(&this.m_decoded_input, offset, tokenizer_current_byte_offset(this) - offset) else trap("MUST: substring_from_byte_offset_with_shared_superstring")