Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.1

0.6.1 / src / ak / utf16_string.lucb

#==============================================================================================
#
#   utf16_string - Utf16String: a strongly owned sequence of code points encoded as UTF-16
#
#   DESCRIPTION:
#       Port of AK/Utf16String.h (lines 1-375) and AK/Utf16String.cpp (lines 1-203), in the
#       header's order with the .cpp bodies in place of their declarations.
#
#       As a memory optimization, a UTF-16 string that is entirely ASCII is stored as 8-bit
#       bytes: inline for up to seven of them, otherwise in a Utf16StringData. `==` on
#       Utf16String does not compile (its base holds a union): compare with the utf16_string_base
#       functions.
#
#==============================================================================================

# mark: Construction ============================================================================

## Ported from AK/Utf16String.h:34 AK::Utf16String::Utf16String(Utf16StringBase&&).
pub func utf16_string_construct_utf16_string_base(this: Utf16String*, base_: Utf16StringBase):
    this.utf16_string_base = base_

## `Utf16String {}`: the empty string.
pub func utf16_string_new() -> Utf16String:
    return Utf16String(utf16_string_base = utf16_string_base_new())

## A Utf16String over a Utf16StringData (the private NonnullRefPtr constructor).
func utf16_string_from_data(data: const Utf16StringData*) -> Utf16String:
    var string = Utf16String(utf16_string_base = Utf16StringBase())
    utf16_string_base_construct_utf16_string_data(&string.utf16_string_base, data)
    return string

## A Utf16String whose short ASCII storage holds `bytes` (the C++ writes
## `string.m_value.short_ascii_string` and copies into its storage).
func utf16_string_from_short_ascii(bytes: const u8[]) -> Utf16String:
    var short_string = short_string_create_with_byte_count(bytes.length)
    copy_bytes(short_string.storage[..<bytes.length], bytes)
    var string = Utf16String(utf16_string_base = Utf16StringBase())
    utf16_string_base_construct_short_string(&string.utf16_string_base, short_string)
    return string

## Ported from AK/Utf16String.h:39 AK::Utf16String::from_utf8(StringView): `utf8_string` must be
## valid UTF-8.
pub func utf16_string_from_utf8_string_view(utf8_string: StringView) -> Utf16String:
    let utf8 = Utf8View(m_string = utf8_string)
    assert(utf8_view_validate_allow_lonely_surrogates(&utf8))
    return utf16_string_from_utf8_without_validation(utf8_string)

## Ported from AK/Utf16String.h:45 AK::Utf16String::from_utf8(String const&).
pub func utf16_string_from_utf8_string(utf8_string: const String*) -> Utf16String:
    return utf16_string_from_utf8_without_validation(string_bytes_as_string_view(utf8_string))

## Ported from AK/Utf16String.h:50 AK::Utf16String::from_utf8(FlyString const&).
pub func utf16_string_from_utf8_fly_string(utf8_string: const FlyString*) -> Utf16String:
    return utf16_string_from_utf8_without_validation(fly_string_bytes_as_string_view(utf8_string))

## `u"text"_utf16` from a `str`: a Utf16String of a literal or any other valid UTF-8 text.
pub func utf16_string_from_str(text: str) -> Utf16String:
    return utf16_string_from_utf8_without_validation(sv(text))

## Ported from AK/Utf16String.cpp:17 AK::Utf16String::from_utf8_with_replacement_character.
pub func utf16_string_from_utf8_with_replacement_character(utf8_string: StringView, with_bom_handling: Utf16StringWithBomHandling = Utf16StringWithBomHandling.yes) -> Utf16String:
    var input = utf8_string
    let bytes = string_view_bytes(&utf8_string)
    if with_bom_handling == Utf16StringWithBomHandling.yes and bytes.length >= 3 and bytes[0] == 0xef and bytes[1] == 0xbb and bytes[2] == 0xbf:
        input = string_view_substring_view(&utf8_string, 3)

    let utf8_view = Utf8View(m_string = input)

    if utf8_view_validate_allow_lonely_surrogates(&utf8_view, AllowLonelySurrogates.no):
        return utf16_string_from_utf8_without_validation(input)

    var builder = string_builder_new_mode(StringBuilderMode.utf16)

    var it = utf8_view_begin(&utf8_view)
    while not utf8_code_point_iterator_done(&it):
        let code_point = utf8_code_point_iterator_deref(&it)
        if is_unicode_surrogate(code_point):
            string_builder_append_code_point(&builder, unicode_utils_replacement_code_point)
        else:
            string_builder_append_code_point(&builder, code_point)
        _ = utf8_code_point_iterator_inc(&it)

    return string_builder_to_utf16_string(&builder)

## Ported from AK/Utf16String.h:63 AK::Utf16String::try_from_utf8: fails when `utf8_string` is
## not valid UTF-8.
pub func utf16_string_try_from_utf8(utf8_string: StringView) -> Utf16String!:
    let utf8 = Utf8View(m_string = utf8_string)
    if not utf8_view_validate_allow_lonely_surrogates(&utf8):
        error(ak_error, "Input was not valid UTF-8")
    return utf16_string_from_utf8_without_validation(utf8_string)

## Ported from AK/Utf16String.cpp:57 AK::Utf16String::from_utf8_without_validation.
pub func utf16_string_from_utf8_without_validation(utf8_string: StringView) -> Utf16String:
    if string_view_length(&utf8_string) <= max_short_string_byte_count and string_view_is_ascii(&utf8_string):
        return utf16_string_from_short_ascii(string_view_bytes(&utf8_string))

    return utf16_string_from_data(utf16_string_data_from_utf8(utf8_string, Utf16StringDataAllowAsciiStorage.yes))

## Ported from AK/Utf16String.cpp:43 AK::Utf16String::from_ascii_without_validation.
pub func utf16_string_from_ascii_without_validation(ascii_string: const u8[]) -> Utf16String:
    if ascii_string.length <= max_short_string_byte_count:
        return utf16_string_from_short_ascii(ascii_string)

    return utf16_string_from_data(utf16_string_data_from_ascii(ascii_string))

## Ported from AK/Utf16String.cpp:71 AK::Utf16String::from_utf16.
pub func utf16_string_from_utf16(utf16_string: Utf16View) -> Utf16String:
    if utf16_view_length_in_code_units(&utf16_string) <= max_short_string_byte_count and utf16_view_is_ascii(&utf16_string):
        if utf16_view_has_ascii_storage(&utf16_string):
            return utf16_string_from_short_ascii(utf16_view_bytes(&utf16_string))

        var ascii: u8[8]
        let result = simdutf_convert_utf16_to_utf8(utf16_view_utf16_span(&utf16_string), ascii)
        assert(result == utf16_view_length_in_code_units(&utf16_string))
        return utf16_string_from_short_ascii(ascii[..<result])

    return utf16_string_from_data(utf16_string_data_from_utf16(utf16_string))

## Ported from AK/Utf16String.cpp:91 AK::Utf16String::from_utf32.
pub func utf16_string_from_utf32(utf32_string: Utf32View) -> Utf16String:
    if utf32_view_length(&utf32_string) <= max_short_string_byte_count and utf32_view_is_ascii(&utf32_string):
        var ascii: u8[8]
        let result = simdutf_convert_utf32_to_utf8(utf32_view_span(&utf32_string), ascii)
        assert(result == utf32_view_length(&utf32_string))
        return utf16_string_from_short_ascii(ascii[..<result])

    return utf16_string_from_data(utf16_string_data_from_utf32(utf32_string))

## Ported from AK/Utf16String.h:92 AK::Utf16String::from_ascii_character.
pub func utf16_string_from_ascii_character(character: u8) -> Utf16String:
    var bytes: u8[1] = [character]
    return utf16_string_from_short_ascii(bytes)

## Ported from AK/Utf16String.h:99 AK::Utf16String::from_ascii_short_string_without_validation(char const*, size_t).
pub func utf16_string_from_ascii_short_string_without_validation_u8(data: const u8[]) -> Utf16String:
    assert(data.length <= max_short_string_byte_count)
    return utf16_string_from_short_ascii(data)

## Ported from AK/Utf16String.h:108 AK::Utf16String::from_ascii_short_string_without_validation(char16_t const*, size_t).
pub func utf16_string_from_ascii_short_string_without_validation_u16(data: const u16[]) -> Utf16String:
    assert(data.length <= max_short_string_byte_count)
    var bytes: u8[8]
    for i in 0..<data.length:
        bytes[i] = (u8)data[i]
    return utf16_string_from_short_ascii(bytes[..<data.length])

## The lambda of Utf16String::from_code_point and repeated: collects the code units.
struct Utf16StringCodeUnits: Utf16CodeUnitCallback:
    var code_units: u16[2]
    var length_in_code_units: usize

    func call(code_unit: u16):
        self.code_units[self.length_in_code_units] = code_unit
        self.length_in_code_units += 1

## Ported from AK/Utf16String.h:117 AK::Utf16String::from_code_point.
pub func utf16_string_from_code_point(code_point: u32) -> Utf16String:
    var code_units = Utf16StringCodeUnits()
    _ = unicode_utils_code_point_to_utf16(code_point, &code_units)

    return utf16_string_from_utf16(utf16_view_from_utf16(code_units.code_units[..<code_units.length_in_code_units]))

## Ported from AK/Utf16String.h:141 AK::Utf16String::number(T) for signed integral T.
pub func utf16_string_number_i64(value: i64) -> Utf16String:
    return create_string_from_number_utf16_string_i64(value)

## Ported from AK/Utf16String.h:141 AK::Utf16String::number(T) for unsigned integral T.
pub func utf16_string_number_u64(value: u64) -> Utf16String:
    return create_string_from_number_utf16_string_u64(value)

## Ported from AK/Utf16String.h:150 AK::Utf16String::join over StringViews with the default "{}"
## format.
pub func utf16_string_join_string_view(separator: StringView, collection: const StringView[]) -> Utf16String:
    var builder = string_builder_new_mode(StringBuilderMode.utf16)
    string_builder_join_string_view(&builder, separator, collection)

    return string_builder_to_utf16_string(&builder)

## Ported from AK/Utf16String.h:150 AK::Utf16String::join over Utf16Views with the default "{}"
## format.
pub func utf16_string_join_utf16_view(separator: Utf16View, collection: const Utf16View[]) -> Utf16String:
    var builder = string_builder_new_mode(StringBuilderMode.utf16)
    string_builder_join_utf16_view(&builder, separator, collection)

    return string_builder_to_utf16_string(&builder)

## Ported from AK/Utf16String.cpp:163 AK::Utf16String::repeated.
pub func utf16_string_repeated(code_point: u32, count: usize) -> Utf16String:
    if count <= max_short_string_byte_count and is_ascii(code_point):
        var bytes: u8[8]
        for i in 0..<count:
            bytes[i] = (u8)code_point
        return utf16_string_from_short_ascii(bytes[..<count])

    var code_units = Utf16StringCodeUnits()
    _ = unicode_utils_code_point_to_utf16(code_point, &code_units)

    var builder = string_builder_new_mode(StringBuilderMode.utf16)
    string_builder_append_repeated_utf16_view(&builder, utf16_view_from_utf16(code_units.code_units[..<code_units.length_in_code_units]), count)
    return string_builder_to_utf16_string(&builder)

# mark: Conversions =============================================================================

## Ported from AK/Utf16String.cpp:186 AK::Utf16String::to_well_formed: lonely surrogates
## replaced by U+FFFD.
pub func utf16_string_to_well_formed(this: const Utf16String*) -> Utf16String:
    let view = utf16_string_base_utf16_view(&this.utf16_string_base)
    if utf16_view_validate(&view):
        return *this
    return utf16_string_from_data(utf16_string_data_to_well_formed(view))

## Ported from AK/Utf16String.cpp:193 AK::Utf16String::to_well_formed_utf8.
pub func utf16_string_to_well_formed_utf8(this: const Utf16String*) -> String:
    let view = utf16_string_base_utf16_view(&this.utf16_string_base)
    if utf16_view_validate(&view):
        return utf16_string_base_to_utf8(&this.utf16_string_base, AllowLonelySurrogates.no)
    let well_formed = utf16_string_to_well_formed(this)
    return utf16_string_base_to_utf8(&well_formed.utf16_string_base, AllowLonelySurrogates.no)

## Ported from AK/Utf16String.h:175 AK::Utf16String::to_ascii_lowercase.
pub func utf16_string_to_ascii_lowercase(this: const Utf16String*) -> Utf16String:
    let view = utf16_string_base_utf16_view(&this.utf16_string_base)

    var has_upper = false
    if utf16_view_has_ascii_storage(&view):
        for byte in utf16_view_ascii_span(&view):
            if is_ascii_upper_alpha(byte):
                has_upper = true
                break
    else:
        for code_unit in utf16_view_utf16_span(&view):
            if is_ascii_upper_alpha(code_unit):
                has_upper = true
                break
    if not has_upper:
        return *this

    return utf16_view_to_ascii_lowercase(&view)

## Ported from AK/Utf16String.h:190 AK::Utf16String::to_ascii_uppercase.
pub func utf16_string_to_ascii_uppercase(this: const Utf16String*) -> Utf16String:
    let view = utf16_string_base_utf16_view(&this.utf16_string_base)

    var has_lower = false
    if utf16_view_has_ascii_storage(&view):
        for byte in utf16_view_ascii_span(&view):
            if is_ascii_lower_alpha(byte):
                has_lower = true
                break
    else:
        for code_unit in utf16_view_utf16_span(&view):
            if is_ascii_lower_alpha(code_unit):
                has_lower = true
                break
    if not has_lower:
        return *this

    return utf16_view_to_ascii_uppercase(&view)

## Ported from AK/Utf16String.h:205 AK::Utf16String::to_ascii_titlecase.
pub func utf16_string_to_ascii_titlecase(this: const Utf16String*) -> Utf16String:
    let view = utf16_string_base_utf16_view(&this.utf16_string_base)
    return utf16_view_to_ascii_titlecase(&view)

## Ported from AK/Utf16String.h:210 AK::Utf16String::replace(char16_t, ...).
pub func utf16_string_replace_u16(this: const Utf16String*, needle: u16, replacement: Utf16View, replace_mode: ReplaceMode) -> Utf16String:
    let view = utf16_string_base_utf16_view(&this.utf16_string_base)
    if utf16_view_is_empty(&view) or not utf16_view_contains_u16(&view, needle):
        return *this

    return utf16_view_replace_u16(&view, needle, replacement, replace_mode)

## Ported from AK/Utf16String.h:219 AK::Utf16String::replace(Utf16View const&, ...).
pub func utf16_string_replace_utf16_view(this: const Utf16String*, needle: Utf16View, replacement: Utf16View, replace_mode: ReplaceMode) -> Utf16String:
    let view = utf16_string_base_utf16_view(&this.utf16_string_base)
    if utf16_view_is_empty(&view) or not utf16_view_contains_utf16_view(&view, needle):
        return *this

    return utf16_view_replace_utf16_view(&view, needle, replacement, replace_mode)

## Ported from AK/Utf16String.h:228 AK::Utf16String::trim.
pub func utf16_string_trim(this: const Utf16String*, code_units: Utf16View, mode: TrimMode = TrimMode.both) -> Utf16String:
    if utf16_string_base_is_empty(&this.utf16_string_base):
        return utf16_string_new()

    var needs_trimming = false

    if mode == TrimMode.left or mode == TrimMode.both:
        needs_trimming = needs_trimming or utf16_view_contains_u16(&code_units, utf16_string_base_code_unit_at(&this.utf16_string_base, 0))
    if mode == TrimMode.right or mode == TrimMode.both:
        needs_trimming = needs_trimming or utf16_view_contains_u16(&code_units, utf16_string_base_code_unit_at(&this.utf16_string_base, utf16_string_base_length_in_code_units(&this.utf16_string_base) - 1))

    if not needs_trimming:
        return *this

    let view = utf16_string_base_utf16_view(&this.utf16_string_base)
    return utf16_string_from_utf16(utf16_view_trim(&view, code_units, mode))

## Ported from AK/Utf16String.h:246 AK::Utf16String::trim_ascii_whitespace.
pub func utf16_string_trim_ascii_whitespace(this: const Utf16String*, mode: TrimMode = TrimMode.both) -> Utf16String:
    return utf16_string_trim(this, utf16_sv(" \n\t\u{b}\u{c}\r"), mode)

## Ported from AK/Utf16String.h:251 AK::Utf16String::escape_html_entities.
pub func utf16_string_escape_html_entities(this: const Utf16String*) -> Utf16String:
    let view = utf16_string_base_utf16_view(&this.utf16_string_base)
    return utf16_view_escape_html_entities(&view)

## Ported from AK/Utf16String.cpp:110 AK::Utf16String::from_string_builder.
pub func utf16_string_from_string_builder(builder: StringBuilder*) -> Utf16String:
    let view = string_builder_utf16_string_view(builder)

    if utf16_view_length_in_code_units(&view) <= max_short_string_byte_count and utf16_view_has_ascii_storage(&view):
        return utf16_string_from_short_ascii(utf16_view_bytes(&view))

    return utf16_string_from_data(utf16_string_data_from_string_builder(builder))

## Ported from AK/Utf16String.cpp:127 AK::Utf16String::from_ipc_stream.
pub func utf16_string_from_ipc_stream(stream: Stream*, length_in_code_units: usize, is_ascii_: bool) -> Utf16String!:
    if is_ascii_ and length_in_code_units <= max_short_string_byte_count:
        var bytes: u8[8]
        try stream_read_until_filled(stream, bytes[..<length_in_code_units])

        let view = string_view_from_bytes(bytes[..<length_in_code_units])
        if not string_view_is_ascii(&view):
            error(ak_error, "Stream contains invalid ASCII data")

        return utf16_string_from_short_ascii(bytes[..<length_in_code_units])

    return utf16_string_from_data(try utf16_string_data_from_ipc_stream(stream, length_in_code_units, is_ascii_))

## Ported from AK/Utf16String.h:265 AK::Utf16String::Utf16String(NonnullRefPtr<Utf16StringData const>).
pub func utf16_string_construct_utf16_string_data(this: Utf16String*, value: const Utf16StringData*):
    utf16_string_base_construct_utf16_string_data(&this.utf16_string_base, value)

# mark: Literals ================================================================================

## Ported from AK/Utf16String.h:353 operator""_utf16(char const*, size_t): short ASCII literals
## become short strings; anything else must be valid UTF-8.
pub func literalutf16_u8(string: const u8*?, length: usize) -> Utf16String:
    let view = literalsv_u8(string, length)
    # OPTIMIZATION: Short ASCII strings become compile-time constants with no runtime validation
    # or heap allocation.
    if length <= max_short_string_byte_count and string_view_is_ascii(&view):
        return utf16_string_from_ascii_short_string_without_validation_u8(string_view_bytes(&view))

    let utf8 = Utf8View(m_string = view)
    assert(utf8_view_validate_allow_lonely_surrogates(&utf8))
    return utf16_string_from_utf8_without_validation(view)

## Ported from AK/Utf16String.h:367 operator""_utf16(char16_t const*, size_t).
pub func literalutf16_u16(string: const u16*?, length: usize) -> Utf16String:
    let view = literalsv_u16(string, length)
    # OPTIMIZATION: Short ASCII strings become compile-time constants with no runtime work.
    if length <= max_short_string_byte_count and utf16_view_is_ascii(&view):
        return utf16_string_from_ascii_short_string_without_validation_u16(utf16_view_utf16_span(&view))

    return utf16_string_from_utf16(view)