Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.0

0.6.0 / src / web_url / pattern_tokenizer.lucb

#==============================================================================================
#
#   pattern_tokenizer - URLPattern: tokenizing a pattern string
#
#   DESCRIPTION:
#       Port of Libraries/LibURL/Pattern/Tokenizer.cpp (lines 1-440) and
#       Libraries/LibURL/Pattern/Tokenizer.h (lines 1-117). Positions are code point indices,
#       u32 in the tokenizer's helpers as in the C++.
#
#==============================================================================================

# mark: Tokens ==================================================================================

## Ported from Libraries/LibURL/Pattern/Tokenizer.cpp:13 URL::Pattern::Token::type_to_string.
pub func token_type_to_string(type_: TokenType) -> ak.StringView:
    match type_:
        .open: return sv("Open")
        .close: return sv("Close")
        .regexp: return sv("Regexp")
        .name: return sv("Name")
        .char_: return sv("Char")
        .escaped_char: return sv("EscapedChar")
        .other_modifier: return sv("OtherModifier")
        .asterisk: return sv("Asterisk")
        .end: return sv("End")
        .invalid_char: return sv("InvalidChar")
        _: trap("VERIFY_NOT_REACHED")

## Ported from Libraries/LibURL/Pattern/Tokenizer.cpp:40 URL::Pattern::Token::to_string.
pub func token_to_string(this: const Token*) -> ak.String:
    # String::formatted("{}, index: {}, value: '{}'", type_to_string(type), index, value)
    var builder: ak.StringBuilder
    ak.string_builder_construct(&builder)
    ak.string_builder_append_string_view(&builder, token_type_to_string(this.type_))
    append_literal(&builder, ", index: ")
    append_decimal(&builder, this.index)
    append_literal(&builder, ", value: '")
    append_string(&builder, this.value)
    ak.string_builder_append_u8(&builder, '\'')
    return ak.string_builder_to_string(&builder) else trap("MUST: String::formatted")

# mark: Tokenizing ==============================================================================

## Ported from Libraries/LibURL/Pattern/Tokenizer.cpp:45 URL::Pattern::Tokenizer::Tokenizer.
pub func tokenizer_construct(this: Tokenizer*, input: ak.Utf8View, policy: TokenizerPolicy):
    this.m_input = input
    this.m_policy = policy

# tokenizer’s input's code point length.
func tokenizer_input_length(this: const Tokenizer*) -> usize:
    return ak.utf8_view_length(&this.m_input)

## Ported from Libraries/LibURL/Pattern/Tokenizer.cpp:52 URL::Pattern::Tokenizer::tokenize.
# https://urlpattern.spec.whatwg.org/#tokenize
pub func tokenizer_tokenize(input: ak.Utf8View, policy: TokenizerPolicy) -> ak.Vector[Token]!:
    assert(ak.utf8_view_validate_allow_lonely_surrogates(&input))

    # 1. Let tokenizer be a new tokenizer.
    # 2. Set tokenizer’s input to input.
    # 3. Set tokenizer’s policy to policy.
    var tokenizer: Tokenizer
    tokenizer_construct(&tokenizer, input, policy)

    # 4. While tokenizer’s index is less than tokenizer’s input's code point length:
    while tokenizer.m_index < tokenizer_input_length(&tokenizer):
        # 1. Run seek and get the next code point given tokenizer and tokenizer’s index.
        tokenizer_seek_and_get_the_next_code_point(&tokenizer, (u32)tokenizer.m_index)

        # 2. If tokenizer’s code point is U+002A (*):
        if tokenizer.m_code_point == (u32)'*':
            # 1. Run add a token with default position and length given tokenizer and "asterisk".
            tokenizer_add_a_token_with_default_position_and_length(&tokenizer, TokenType.asterisk)

            # 2. Continue.
            continue

        # 3. If tokenizer’s code point is U+002B (+) or U+003F (?):
        if tokenizer.m_code_point == (u32)'+' or tokenizer.m_code_point == (u32)'?':
            # 1. Run add a token with default position and length given tokenizer and "other-modifier".
            tokenizer_add_a_token_with_default_position_and_length(&tokenizer, TokenType.other_modifier)

            # 2. Continue.
            continue

        # 4. If tokenizer’s code point is U+005C (\):
        if tokenizer.m_code_point == (u32)'\\':
            # 1. If tokenizer’s index is equal to tokenizer’s input's code point length − 1:
            if tokenizer.m_index == tokenizer_input_length(&tokenizer) - 1:
                # 1. Run process a tokenizing error given tokenizer, tokenizer’s next index, and tokenizer’s index.
                try tokenizer_process_a_tokenizing_error(&tokenizer, (u32)tokenizer.m_next_index, (u32)tokenizer.m_index)

                # 2. Continue.
                continue

            # 2. Let escaped index be tokenizer’s next index.
            let escaped_index = tokenizer.m_next_index

            # 3. Run get the next code point given tokenizer.
            tokenizer_get_the_next_code_point(&tokenizer)

            # 4. Run add a token with default length given tokenizer, "escaped-char", tokenizer’s next index, and escaped index.
            tokenizer_add_a_token_with_default_length(&tokenizer, TokenType.escaped_char, (u32)tokenizer.m_next_index, (u32)escaped_index)

            # 5. Continue.
            continue

        # 5. If tokenizer’s code point is U+007B ({):
        if tokenizer.m_code_point == (u32)'{':
            # 1. Run add a token with default position and length given tokenizer and "open".
            tokenizer_add_a_token_with_default_position_and_length(&tokenizer, TokenType.open)

            # 2. Continue.
            continue

        # 6. If tokenizer’s code point is U+007D (}):
        if tokenizer.m_code_point == (u32)'}':
            # 1. Run add a token with default position and length given tokenizer and "close".
            tokenizer_add_a_token_with_default_position_and_length(&tokenizer, TokenType.close)

            # 2. Continue.
            continue

        # 1. If tokenizer’s code point is U+003A (:):
        if tokenizer.m_code_point == (u32)':':
            # 1. Let name position be tokenizer’s next index.
            var name_position = tokenizer.m_next_index

            # 2. Let name start be name position.
            let name_start = name_position

            # 3. While name position is less than tokenizer’s input's code point length:
            while name_position < tokenizer_input_length(&tokenizer):
                # 1. Run seek and get the next code point given tokenizer and name position.
                tokenizer_seek_and_get_the_next_code_point(&tokenizer, (u32)name_position)

                # 2. Let first code point be true if name position equals name start and false otherwise.
                let first_code_point_flag = name_position == name_start

                # 3. Let valid code point be the result of running is a valid name code point given tokenizer’s code point and first code point.
                let valid_code_point = tokenizer_is_a_valid_name_code_point(tokenizer.m_code_point, first_code_point_flag)

                # 4. If valid code point is false break.
                if not valid_code_point:
                    break

                # 5. Set name position to tokenizer’s next index.
                name_position = tokenizer.m_next_index

            # 4. If name position is less than or equal to name start:
            if name_position <= name_start:
                # 1. Run process a tokenizing error given tokenizer, name start, and tokenizer’s index.
                try tokenizer_process_a_tokenizing_error(&tokenizer, (u32)name_start, (u32)tokenizer.m_index)

                # 2. Continue.
                continue

            # 5. Run add a token with default length given tokenizer, "name", name position, and name start.
            tokenizer_add_a_token_with_default_length(&tokenizer, TokenType.name, (u32)name_position, (u32)name_start)

            # 6. Continue.
            continue

        # 8. If tokenizer’s code point is U+0028 (():
        if tokenizer.m_code_point == (u32)'(':
            # 1. Let depth be 1.
            var depth: u32 = 1

            # 2. Let regexp position be tokenizer’s next index.
            var regexp_position = tokenizer.m_next_index

            # 3. Let regexp start be regexp position.
            let regexp_start = regexp_position

            # 4. Let error be false.
            var error_seen = false

            # 5. While regexp position is less than tokenizer’s input's code point length:
            while regexp_position < tokenizer_input_length(&tokenizer):
                # 1. Run seek and get the next code point given tokenizer and regexp position.
                tokenizer_seek_and_get_the_next_code_point(&tokenizer, (u32)regexp_position)

                # 2. If the result of running is ASCII given tokenizer’s code point is false:
                if not ak.is_ascii(tokenizer.m_code_point):
                    # 1. Run process a tokenizing error given tokenizer, regexp start, and tokenizer’s index.
                    try tokenizer_process_a_tokenizing_error(&tokenizer, (u32)regexp_start, (u32)tokenizer.m_index)

                    # 2. Set error to true.
                    error_seen = true

                    # 3. Break.
                    break

                # 3. If regexp position equals regexp start and tokenizer’s code point is U+003F (?):
                if regexp_position == regexp_start and tokenizer.m_code_point == (u32)'?':
                    # 1. Run process a tokenizing error given tokenizer, regexp start, and tokenizer’s index.
                    try tokenizer_process_a_tokenizing_error(&tokenizer, (u32)regexp_start, (u32)tokenizer.m_index)

                    # 2. Set error to true.
                    error_seen = true

                    # 3. Break.
                    break

                # 4. If tokenizer’s code point is U+005C (\):
                if tokenizer.m_code_point == (u32)'\\':
                    # 1. If regexp position equals tokenizer’s input's code point length − 1:
                    if regexp_position == tokenizer_input_length(&tokenizer) - 1:
                        # 1. Run process a tokenizing error given tokenizer, regexp start, and tokenizer’s index.
                        try tokenizer_process_a_tokenizing_error(&tokenizer, (u32)regexp_start, (u32)tokenizer.m_index)

                        # 2. Set error to true.
                        error_seen = true

                        # 3. Break
                        break

                    # 2. Run get the next code point given tokenizer.
                    tokenizer_get_the_next_code_point(&tokenizer)

                    # 3. If the result of running is ASCII given tokenizer’s code point is false:
                    if not ak.is_ascii(tokenizer.m_code_point):
                        # 1. Run process a tokenizing error given tokenizer, regexp start, and tokenizer’s index.
                        try tokenizer_process_a_tokenizing_error(&tokenizer, (u32)regexp_start, (u32)tokenizer.m_index)

                        # 2. Set error to true.
                        error_seen = true

                        # 3. Break.
                        break

                    # 4. Set regexp position to tokenizer’s next index.
                    regexp_position = tokenizer.m_next_index

                    # 5. Continue.
                    continue

                # 5. If tokenizer’s code point is U+0029 ()):
                if tokenizer.m_code_point == (u32)')':
                    # 1. Decrement depth by 1.
                    depth -= 1

                    # 1. If depth is 0:
                    if depth == 0:
                        # 1. Set regexp position to tokenizer’s next index.
                        regexp_position = tokenizer.m_next_index

                        # 2. Break.
                        break
                # 6. Otherwise if tokenizer’s code point is U+0028 (():
                elif tokenizer.m_code_point == (u32)'(':
                    # 1. Increment depth by 1.
                    depth += 1

                    # 2. If regexp position equals tokenizer’s input's code point length − 1:
                    if regexp_position == tokenizer_input_length(&tokenizer) - 1:
                        # 1. Run process a tokenizing error given tokenizer, regexp start, and tokenizer’s index.
                        try tokenizer_process_a_tokenizing_error(&tokenizer, (u32)regexp_start, (u32)tokenizer.m_index)

                        # 2. Set error to true.
                        error_seen = true

                        # 3. Break
                        break

                    # 3. Let temporary position be tokenizer’s next index.
                    let temporary_position = tokenizer.m_next_index

                    # 4. Run get the next code point given tokenizer.
                    tokenizer_get_the_next_code_point(&tokenizer)

                    # 5. If tokenizer’s code point is not U+003F (?):
                    if tokenizer.m_code_point != (u32)'?':
                        # 1. Run process a tokenizing error given tokenizer, regexp start, and tokenizer’s index.
                        try tokenizer_process_a_tokenizing_error(&tokenizer, (u32)regexp_start, (u32)tokenizer.m_index)

                        # 2. Set error to true.
                        error_seen = true

                        # 3. Break.
                        break

                    # 6. Set tokenizer’s next index to temporary position.
                    tokenizer.m_next_index = temporary_position

                # 7. Set regexp position to tokenizer’s next index.
                regexp_position = tokenizer.m_next_index

            # 6. If error is true continue.
            if error_seen:
                continue

            # 7. If depth is not zero:
            if depth != 0:
                # 1. Run process a tokenizing error given tokenizer, regexp start, and tokenizer’s index.
                try tokenizer_process_a_tokenizing_error(&tokenizer, (u32)regexp_start, (u32)tokenizer.m_index)

                # 2. Continue.
                continue

            # 8. Let regexp length be regexp position − regexp start − 1.
            let regexp_length = regexp_position - regexp_start - 1

            # 9. If regexp length is zero:
            if regexp_length == 0:
                # 1. Run process a tokenizing error given tokenizer, regexp start, and tokenizer’s index.
                try tokenizer_process_a_tokenizing_error(&tokenizer, (u32)regexp_start, (u32)tokenizer.m_index)

                # 2. Continue.
                continue

            # 10. Run add a token given tokenizer, "regexp", regexp position, regexp start, and regexp length.
            tokenizer_add_a_token(&tokenizer, TokenType.regexp, (u32)regexp_position, (u32)regexp_start, (u32)regexp_length)

            # 11. Continue.
            continue

        # 9. Run add a token with default position and length given tokenizer and "char".
        tokenizer_add_a_token_with_default_position_and_length(&tokenizer, TokenType.char_)

    # 5. Run add a token with default length given tokenizer, "end", tokenizer’s index, and tokenizer’s index.
    tokenizer_add_a_token_with_default_length(&tokenizer, TokenType.end, (u32)tokenizer.m_index, (u32)tokenizer.m_index)

    # 6. Return tokenizer’s token list.
    return tokenizer.m_token_list

## Ported from Libraries/LibURL/Pattern/Tokenizer.cpp:355 URL::Pattern::Tokenizer::get_the_next_code_point.
# https://urlpattern.spec.whatwg.org/#get-the-next-code-point
pub func tokenizer_get_the_next_code_point(this: Tokenizer*):
    # 1. Set tokenizer’s code point to the Unicode code point in tokenizer’s input at the position indicated by tokenizer’s next index.
    let substring = ak.utf8_view_unicode_substring_view_usize(&this.m_input, this.m_next_index, 1)
    let begin = ak.utf8_view_begin(&substring)
    this.m_code_point = ak.utf8_code_point_iterator_deref(&begin)

    # 2. Increment tokenizer’s next index by 1.
    this.m_next_index += 1

## Ported from Libraries/LibURL/Pattern/Tokenizer.cpp:365 URL::Pattern::Tokenizer::seek_and_get_the_next_code_point.
# https://urlpattern.spec.whatwg.org/#seek-and-get-the-next-code-point
pub func tokenizer_seek_and_get_the_next_code_point(this: Tokenizer*, index: u32):
    # 1. Set tokenizer’s next index to index.
    this.m_next_index = index

    # 2. Run get the next code point given tokenizer.
    tokenizer_get_the_next_code_point(this)

## Ported from Libraries/LibURL/Pattern/Tokenizer.cpp:375 URL::Pattern::Tokenizer::add_a_token.
# https://urlpattern.spec.whatwg.org/#add-a-token
pub func tokenizer_add_a_token(this: Tokenizer*, type_: TokenType, next_position: u32, value_position: u32, value_length: u32):
    # 1. Let token be a new token.
    var token: Token
    token_init_fields(&token)

    # 2. Set token’s type to type.
    token.type_ = type_

    # 3. Set token’s index to tokenizer’s index.
    token.index = (u32)this.m_index

    # 4. Set token’s value to the code point substring from value position with length value length within tokenizer’s input.
    let substring = ak.utf8_view_unicode_substring_view_usize(&this.m_input, value_position, value_length)
    token.value = ak.string_from_utf8(ak.utf8_view_as_string(&substring)) else trap("MUST: String::from_utf8")

    # 5. Append token to the back of tokenizer’s token list.
    ak.vector_append_t(&this.m_token_list, token)

    # 5. Set tokenizer’s index to next position.
    this.m_index = next_position

## Ported from Libraries/LibURL/Pattern/Tokenizer.cpp:397 URL::Pattern::Tokenizer::add_a_token_with_default_length.
# https://urlpattern.spec.whatwg.org/#add-a-token-with-default-length
pub func tokenizer_add_a_token_with_default_length(this: Tokenizer*, type_: TokenType, next_position: u32, value_position: u32):
    # 1. Let computed length be next position − value position.
    let computed_length = next_position - value_position

    # 2. Run add a token given tokenizer, type, next position, value position, and computed length.
    tokenizer_add_a_token(this, type_, next_position, value_position, computed_length)

## Ported from Libraries/LibURL/Pattern/Tokenizer.cpp:407 URL::Pattern::Tokenizer::add_a_token_with_default_position_and_length.
# https://urlpattern.spec.whatwg.org/#add-a-token-with-default-position-and-length
pub func tokenizer_add_a_token_with_default_position_and_length(this: Tokenizer*, type_: TokenType):
    # 1. Run add a token with default length given tokenizer, type, tokenizer’s next index, and tokenizer’s index.
    tokenizer_add_a_token_with_default_length(this, type_, (u32)this.m_next_index, (u32)this.m_index)

## Ported from Libraries/LibURL/Pattern/Tokenizer.cpp:414 URL::Pattern::Tokenizer::process_a_tokenizing_error.
# https://urlpattern.spec.whatwg.org/#process-a-tokenizing-error
pub func tokenizer_process_a_tokenizing_error(this: Tokenizer*, next_position: u32, value_position: u32) -> !:
    # 1. If tokenizer’s policy is "strict", then throw a TypeError.
    if this.m_policy == TokenizerPolicy.strict:
        try pattern_throw("Error processing a token")  # FIXME: Improve this error!

    # 2. Assert: tokenizer’s policy is "lenient".
    assert(this.m_policy == TokenizerPolicy.lenient)

    # 3. Run add a token with default length given tokenizer, "invalid-char", next position, and value position.
    tokenizer_add_a_token_with_default_length(this, TokenType.invalid_char, next_position, value_position)

## Ported from Libraries/LibURL/Pattern/Tokenizer.cpp:430 URL::Pattern::Tokenizer::is_a_valid_name_code_point.
# https://urlpattern.spec.whatwg.org/#is-a-valid-name-code-point
pub func tokenizer_is_a_valid_name_code_point(code_point: u32, first: bool) -> bool:
    # 1. If first is true return the result of checking if code point is contained in the IdentifierStart set of code points.
    if first:
        return (code_point == (u32)'$' or code_point == (u32)'_' or web_unicode.code_point_has_identifier_start_property(code_point))

    # 2. Otherwise return the result of checking if code point is contained in the IdentifierPart set of code points.
    return code_point == (u32)'$' or web_unicode.code_point_has_identifier_continue_property(code_point)