Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.1

0.6.1 / src / web_unicode / locale.lucb

#==============================================================================================
#
#   locale - Unicode::Locale: parsing and serializing Unicode locale identifiers
#
#   DESCRIPTION:
#       Port of LibUnicode/Locale.cpp (lines 1-730) and Locale.h (lines 1-156). The parsing is
#       Ladybird's own and ported as it is. The functions the C++ answers with ICU answer from
#       ICU 78.2's data dumped into generated_icu_locales.lucb (available locales, likely
#       subtags, right-to-left scripts); canonicalization puts identifiers into the canonical
#       syntax of UTS #35 (case, order of variants, extensions and keywords) but does not
#       replace deprecated subtags by CLDR's aliases as ICU does.
#
#==============================================================================================

# mark: The lexer ===============================================================================

## `GenericLexer lexer { input }`.
func generic_lexer_over(input: ak.StringView) -> ak.GenericLexer[u8]:
    var lexer: ak.GenericLexer[u8]
    ak.generic_lexer_construct(&lexer, input)
    return lexer

## A StringView of `bytes`.
func view_of_bytes(bytes: const u8[]) -> ak.StringView:
    return ak.StringView(m_characters = bytes.data, m_length = bytes.length)

## MUST(String::from_utf8(view)).
func string_of(view: ak.StringView) -> ak.String:
    return ak.string_from_utf8(view) else trap("MUST: String::from_utf8")

func is_ascii_alpha_byte(c: u8) -> bool:
    return (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z')

func is_ascii_digit_byte(c: u8) -> bool:
    return c >= '0' and c <= '9'

func is_ascii_alphanumeric_byte(c: u8) -> bool:
    return is_ascii_alpha_byte(c) or is_ascii_digit_byte(c)

## all_of(view, predicate)
func all_of(view: ak.StringView, predicate: func(u8) -> bool) -> bool:
    for c in view_bytes(view):
        if not predicate(c):
            return false
    return true

# mark: Subtags =================================================================================

## Ported from Libraries/LibUnicode/Locale.cpp:20 Unicode::is_key.
func is_key(key: ak.StringView) -> bool:
    # key = alphanum alpha
    let bytes = view_bytes(key)
    if bytes.length != 2:
        return false
    return is_ascii_alphanumeric_byte(bytes[0]) and is_ascii_alpha_byte(bytes[1])

## Ported from Libraries/LibUnicode/Locale.cpp:28 Unicode::is_single_type.
func is_single_type(type_: ak.StringView) -> bool:
    # type = alphanum{3,8} (sep alphanum{3,8})*
    # Note: Consecutive types are not handled here, that is left to the caller.
    if type_.m_length < 3 or type_.m_length > 8:
        return false
    return all_of(type_, is_ascii_alphanumeric_byte)

## Ported from Libraries/LibUnicode/Locale.cpp:37 Unicode::is_attribute.
func is_attribute(type_: ak.StringView) -> bool:
    # attribute = alphanum{3,8}
    if type_.m_length < 3 or type_.m_length > 8:
        return false
    return all_of(type_, is_ascii_alphanumeric_byte)

## Ported from Libraries/LibUnicode/Locale.cpp:45 Unicode::is_transformed_key.
func is_transformed_key(key: ak.StringView) -> bool:
    # tkey = alpha digit
    let bytes = view_bytes(key)
    if bytes.length != 2:
        return false
    return is_ascii_alpha_byte(bytes[0]) and is_ascii_digit_byte(bytes[1])

## Ported from Libraries/LibUnicode/Locale.cpp:53 Unicode::is_single_transformed_value.
func is_single_transformed_value(value: ak.StringView) -> bool:
    # tvalue = (sep alphanum{3,8})+
    # Note: Consecutive values are not handled here, that is left to the caller.
    if value.m_length < 3 or value.m_length > 8:
        return false
    return all_of(value, is_ascii_alphanumeric_byte)

## Ported from Libraries/LibUnicode/Locale.cpp:62 Unicode::consume_next_segment.
func consume_next_segment(lexer: ak.GenericLexer[u8]*, with_separator: bool = true) -> ak.StringView?:
    let is_separator = ak.is_any_of(ak.sv("-_"))

    if with_separator:
        if not ak.generic_lexer_next_is_tt_predicate(lexer, &is_separator):
            return none
        ak.generic_lexer_ignore(lexer)

    let segment = ak.generic_lexer_consume_until_tt_predicate(lexer, &is_separator)
    if segment.m_length == 0:
        ak.generic_lexer_retreat_usize(lexer, 1 if with_separator else 0)
        return none

    return segment

## Ported from Libraries/LibUnicode/Locale.cpp:81 Unicode::is_type_identifier.
pub func is_type_identifier(identifier: ak.StringView) -> bool:
    # type = alphanum{3,8} (sep alphanum{3,8})*
    var lexer = generic_lexer_over(identifier)

    while true:
        let type_ = consume_next_segment(&lexer, ak.generic_lexer_tell(&lexer) > 0) else break
        if not is_single_type(type_):
            return false

    return ak.generic_lexer_is_eof(&lexer) and ak.generic_lexer_tell(&lexer) > 0

## Ported from Libraries/LibUnicode/Locale.h:101 Unicode::is_unicode_language_subtag.
# Note: These methods only verify that the provided strings match the EBNF grammar of the
# Unicode identifier subtag (i.e. no validation is done that the tags actually exist).
pub func is_unicode_language_subtag(subtag: ak.StringView) -> bool:
    # unicode_language_subtag = alpha{2,3} | alpha{5,8}
    if subtag.m_length < 2 or subtag.m_length == 4 or subtag.m_length > 8:
        return false
    return all_of(subtag, is_ascii_alpha_byte)

## Ported from Libraries/LibUnicode/Locale.h:109 Unicode::is_unicode_script_subtag.
pub func is_unicode_script_subtag(subtag: ak.StringView) -> bool:
    # unicode_script_subtag = alpha{4}
    if subtag.m_length != 4:
        return false
    return all_of(subtag, is_ascii_alpha_byte)

## Ported from Libraries/LibUnicode/Locale.h:117 Unicode::is_unicode_region_subtag.
pub func is_unicode_region_subtag(subtag: ak.StringView) -> bool:
    # unicode_region_subtag = (alpha{2} | digit{3})
    if subtag.m_length == 2:
        return all_of(subtag, is_ascii_alpha_byte)
    if subtag.m_length == 3:
        return all_of(subtag, is_ascii_digit_byte)
    return false

## Ported from Libraries/LibUnicode/Locale.h:127 Unicode::is_unicode_variant_subtag.
pub func is_unicode_variant_subtag(subtag: ak.StringView) -> bool:
    # unicode_variant_subtag = (alphanum{5,8} | digit alphanum{3})
    let bytes = view_bytes(subtag)
    if bytes.length >= 5 and bytes.length <= 8:
        return all_of(subtag, is_ascii_alphanumeric_byte)
    if bytes.length == 4:
        return is_ascii_digit_byte(bytes[0]) and all_of(view_of_bytes(bytes[1..]), is_ascii_alphanumeric_byte)
    return false

# mark: Parsing =================================================================================

# The states of parse_unicode_language_id.
enum LanguageIdParseState:
    parsing_language_or_script
    parsing_script
    parsing_region
    parsing_variant
    done

## Ported from Libraries/LibUnicode/Locale.cpp:97 Unicode::parse_unicode_language_id.
func parse_unicode_language_id_generic_lexer(lexer: ak.GenericLexer[u8]*) -> LanguageId?:
    # https://unicode.org/reports/tr35/#Unicode_language_identifier
    #
    # unicode_language_id = "root"
    #     OR
    # unicode_language_id = ((unicode_language_subtag (sep unicode_script_subtag)?) | unicode_script_subtag)
    #                       (sep unicode_region_subtag)?
    #                       (sep unicode_variant_subtag)*
    var language_id = LanguageId()

    if ak.generic_lexer_consume_specific_string_view(lexer, ak.sv("root")):
        language_id.is_root = true
        return language_id

    var state = LanguageIdParseState.parsing_language_or_script

    while not ak.generic_lexer_is_eof(lexer) and state != LanguageIdParseState.done:
        let segment = consume_next_segment(lexer, state != LanguageIdParseState.parsing_language_or_script) else return none

        # The C++ switch falls through from ParsingScript to ParsingRegion to ParsingVariant.
        if state == LanguageIdParseState.parsing_language_or_script:
            if is_unicode_language_subtag(segment):
                state = LanguageIdParseState.parsing_script
                language_id.language = string_of(segment)
            elif is_unicode_script_subtag(segment):
                state = LanguageIdParseState.parsing_region
                language_id.script = string_of(segment)
            else:
                return none
            continue

        if state == LanguageIdParseState.parsing_script:
            if is_unicode_script_subtag(segment):
                state = LanguageIdParseState.parsing_region
                language_id.script = string_of(segment)
                continue

            state = LanguageIdParseState.parsing_region
        # [[fallthrough]];

        if state == LanguageIdParseState.parsing_region:
            if is_unicode_region_subtag(segment):
                state = LanguageIdParseState.parsing_variant
                language_id.region = string_of(segment)
                continue

            state = LanguageIdParseState.parsing_variant
        # [[fallthrough]];

        # ParseState::ParsingVariant
        if is_unicode_variant_subtag(segment):
            ak.vector_append_t(&language_id.variants, string_of(segment))
        else:
            ak.generic_lexer_retreat_usize(lexer, segment.m_length + 1)
            state = LanguageIdParseState.done

    return language_id

# The states of parse_unicode_locale_extension.
enum LocaleExtensionParseState:
    parsing_attribute_or_keyword
    parsing_attribute
    parsing_keyword
    done

## Join `values` with '-' into a String (StringBuilder::join).
func join_segments(values: const ak.StringView[]) -> ak.String:
    var builder: ak.StringBuilder
    ak.string_builder_construct(&builder)
    for index in 0..<values.length:
        if index > 0:
            ak.string_builder_append_u8(&builder, '-')
        ak.string_builder_append_string_view(&builder, values[index])
    return ak.string_builder_to_string(&builder) else trap("MUST: StringBuilder::to_string")

## Ported from Libraries/LibUnicode/Locale.cpp:178 Unicode::parse_unicode_locale_extension.
func parse_unicode_locale_extension(lexer: ak.GenericLexer[u8]*) -> LocaleExtension?:
    # https://unicode.org/reports/tr35/#unicode_locale_extensions
    #
    # unicode_locale_extensions = sep [uU] ((sep keyword)+ | (sep attribute)+ (sep keyword)*)
    var locale_extension = LocaleExtension()

    var state = LocaleExtensionParseState.parsing_attribute_or_keyword

    while not ak.generic_lexer_is_eof(lexer) and state != LocaleExtensionParseState.done:
        let segment = consume_next_segment(lexer) else return none

        if state == LocaleExtensionParseState.parsing_attribute_or_keyword:
            state = LocaleExtensionParseState.parsing_keyword if is_key(segment) else LocaleExtensionParseState.parsing_attribute

        if state == LocaleExtensionParseState.parsing_attribute:
            if is_attribute(segment):
                ak.vector_append_t(&locale_extension.attributes, string_of(segment))
                continue

            state = LocaleExtensionParseState.parsing_keyword
        # [[fallthrough]];

        # ParseState::ParsingKeyword:
        # keyword = key (sep type)?
        var keyword = Keyword(key = string_of(segment))
        var keyword_values: ak.Vector[ak.StringView]

        if not is_key(segment):
            ak.generic_lexer_retreat_usize(lexer, segment.m_length + 1)
            state = LocaleExtensionParseState.done
            continue

        while true:
            let type_ = consume_next_segment(lexer)

            if type_ == none or not is_single_type(type_ else ak.StringView()):
                if let found = type_:
                    ak.generic_lexer_retreat_usize(lexer, found.m_length + 1)
                break

            ak.vector_append_t(&keyword_values, type_ else break)

        keyword.value = join_segments(ak.vector_span_const(&keyword_values))

        ak.vector_append_t(&locale_extension.keywords, keyword)

    if locale_extension.attributes.m_size == 0 and locale_extension.keywords.m_size == 0:
        return none
    return locale_extension

# The states of parse_transformed_extension.
enum TransformedExtensionParseState:
    parsing_language_or_field
    parsing_language
    parsing_field
    done

## Ported from Libraries/LibUnicode/Locale.cpp:253 Unicode::parse_transformed_extension.
func parse_transformed_extension(lexer: ak.GenericLexer[u8]*) -> TransformedExtension?:
    # https://unicode.org/reports/tr35/#transformed_extensions
    #
    # transformed_extensions = sep [tT] ((sep tlang (sep tfield)*) | (sep tfield)+)
    var transformed_extension = TransformedExtension()

    var state = TransformedExtensionParseState.parsing_language_or_field

    while not ak.generic_lexer_is_eof(lexer) and state != TransformedExtensionParseState.done:
        let segment = consume_next_segment(lexer) else return none

        if state == TransformedExtensionParseState.parsing_language_or_field:
            state = TransformedExtensionParseState.parsing_language if is_unicode_language_subtag(segment) else TransformedExtensionParseState.parsing_field

        if state == TransformedExtensionParseState.parsing_language:
            ak.generic_lexer_retreat_usize(lexer, segment.m_length)

            if let language_id = parse_unicode_language_id_generic_lexer(lexer):
                transformed_extension.language = language_id
                state = TransformedExtensionParseState.parsing_field
                continue

            return none

        # ParseState::ParsingField:
        # tfield = tkey tvalue;
        var field = TransformedField(key = string_of(segment))
        var field_values: ak.Vector[ak.StringView]

        if not is_transformed_key(segment):
            ak.generic_lexer_retreat_usize(lexer, segment.m_length + 1)
            state = TransformedExtensionParseState.done
            continue

        while true:
            let value = consume_next_segment(lexer)

            if value == none or not is_single_transformed_value(value else ak.StringView()):
                if let found = value:
                    ak.generic_lexer_retreat_usize(lexer, found.m_length + 1)
                break

            ak.vector_append_t(&field_values, value else break)

        if ak.vector_size(&field_values) == 0:
            return none

        field.value = join_segments(ak.vector_span_const(&field_values))

        ak.vector_append_t(&transformed_extension.fields, field)

    if transformed_extension.language == none and transformed_extension.fields.m_size == 0:
        return none
    return transformed_extension

## Ported from Libraries/LibUnicode/Locale.cpp:333 Unicode::parse_other_extension.
func parse_other_extension(key: u8, lexer: ak.GenericLexer[u8]*) -> OtherExtension?:
    # https://unicode.org/reports/tr35/#other_extensions
    #
    # other_extensions = sep [alphanum-[tTuUxX]] (sep alphanum{2,8})+ ;
    var other_extension = OtherExtension(key = key)
    var other_values: ak.Vector[ak.StringView]

    if not is_ascii_alphanumeric_byte(key) or key == 'x' or key == 'X':
        return none

    while true:
        let segment = consume_next_segment(lexer) else break

        if segment.m_length < 2 or segment.m_length > 8 or not all_of(segment, is_ascii_alphanumeric_byte):
            ak.generic_lexer_retreat_usize(lexer, segment.m_length + 1)
            break

        ak.vector_append_t(&other_values, segment)

    if ak.vector_size(&other_values) == 0:
        return none

    other_extension.value = join_segments(ak.vector_span_const(&other_values))

    return other_extension

## Ported from Libraries/LibUnicode/Locale.cpp:367 Unicode::parse_extension.
func parse_extension(lexer: ak.GenericLexer[u8]*) -> Extension?:
    # https://unicode.org/reports/tr35/#extensions
    #
    # extensions = unicode_locale_extensions | transformed_extensions | other_extensions
    let starting_position = ak.generic_lexer_tell(lexer)

    if let header = consume_next_segment(lexer):
        if header.m_length == 1:
            let key = view_bytes(header)[0]
            match key:
                'u', 'U':
                    if let extension = parse_unicode_locale_extension(lexer):
                        return Extension.locale_extension(value = extension)
                't', 'T':
                    if let extension = parse_transformed_extension(lexer):
                        return Extension.transformed_extension(value = extension)
                _:
                    if let extension = parse_other_extension(key, lexer):
                        return Extension.other_extension(value = extension)

    ak.generic_lexer_retreat_usize(lexer, ak.generic_lexer_tell(lexer) - starting_position)
    return none

## Ported from Libraries/LibUnicode/Locale.cpp:399 Unicode::parse_private_use_extensions.
func parse_private_use_extensions(lexer: ak.GenericLexer[u8]*) -> ak.Vector[ak.String]:
    # https://unicode.org/reports/tr35/#pu_extensions
    #
    # pu_extensions = = sep [xX] (sep alphanum{1,8})+ ;
    let starting_position = ak.generic_lexer_tell(lexer)

    var none_: ak.Vector[ak.String]
    let header = consume_next_segment(lexer) else return none_

    let header_bytes = view_bytes(header)
    if header_bytes.length == 1 and (header_bytes[0] == 'x' or header_bytes[0] == 'X'):
        # parse_values
        var extensions: ak.Vector[ak.String]

        while true:
            let segment = consume_next_segment(lexer) else break

            if segment.m_length < 1 or segment.m_length > 8 or not all_of(segment, is_ascii_alphanumeric_byte):
                ak.generic_lexer_retreat_usize(lexer, segment.m_length + 1)
                break

            ak.vector_append_t(&extensions, string_of(segment))

        if extensions.m_size != 0:
            return extensions

    ak.generic_lexer_retreat_usize(lexer, ak.generic_lexer_tell(lexer) - starting_position)
    return none_

## Ported from Libraries/LibUnicode/Locale.cpp:438 Unicode::parse_unicode_language_id.
pub func parse_unicode_language_id_string_view(language: ak.StringView) -> LanguageId?:
    var lexer = generic_lexer_over(language)

    let language_id = parse_unicode_language_id_generic_lexer(&lexer)
    if not ak.generic_lexer_is_eof(&lexer):
        return none

    return language_id

## Ported from Libraries/LibUnicode/Locale.cpp:449 Unicode::parse_unicode_locale_id.
pub func parse_unicode_locale_id(locale: ak.StringView) -> LocaleId?:
    var lexer = generic_lexer_over(locale)

    # https://unicode.org/reports/tr35/#Unicode_locale_identifier
    #
    # unicode_locale_id = unicode_language_id
    #                     extensions*
    #                     pu_extensions?
    let language_id = parse_unicode_language_id_generic_lexer(&lexer) else return none

    var locale_id = LocaleId(language_id = language_id)

    while true:
        let extension = parse_extension(&lexer) else break
        ak.vector_append_t(&locale_id.extensions, extension)

    locale_id.private_use_extensions = parse_private_use_extensions(&lexer)

    if not ak.generic_lexer_is_eof(&lexer):
        return none

    return locale_id