Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.1

0.6.1 / src / ak / tests_utf16_string_1.lucb

#==============================================================================================
#
#   tests_utf16_string_1 - Utf16String tests (creation from UTF-8, UTF-16, UTF-32, builders)
#
#   DESCRIPTION:
#       Port of Tests/AK/TestUtf16String.cpp (lines 1-525). `"..."_utf16` is
#       utf16_string_from_str; `u"..."_utf16` is t16_lit. Utf16String::formatted is a variadic
#       template the port does not have: its checks are left out (String::number's are kept).
#       The LibUnicode-backed case mappings (to_lowercase/to_uppercase/to_titlecase/to_casefold,
#       lines 526-1027 of the donor) are not in AK and are not ported.
#
#==============================================================================================

# mark: Helpers =================================================================================

## `u"text"_utf16`: the char16_t literal operator over UTF-16 text.
func t16_lit(text: str) -> Utf16String:
    let view = t16(text)
    let units = utf16_view_utf16_span(&view)
    return literalutf16_u16(units.data, units.length)

## The storage and length checks every creation test makes.
func t16s_expect(string: const Utf16String*, is_ascii_: bool, long_ascii: bool, short_ascii: bool, code_units: usize, code_points: usize):
    let base = &string.utf16_string_base
    assert(utf16_string_base_is_empty(base) == (code_units == 0))
    assert(utf16_string_base_is_ascii(base) == is_ascii_)
    assert(utf16_string_base_has_long_ascii_storage(base) == long_ascii)
    assert(utf16_string_base_has_short_ascii_storage(base) == short_ascii)
    assert(utf16_string_base_length_in_code_units(base) == code_units)
    assert(utf16_string_base_length_in_code_points(base) == code_points)

## `EXPECT_EQ(string.ascii_view(), "text"sv)`.
func t16s_ascii_view_is(string: const Utf16String*, text: str) -> bool:
    let view = utf16_string_base_ascii_view(&string.utf16_string_base)
    return string_view_eq_string_view(&view, sv(text))

## A Utf32View of code points (the `to_utf32_view(U"...")` lambda).
func t16_utf32(code_points: const u32[]) -> Utf32View:
    let storage = new u32[code_points.length + 1] else trap("t16_utf32: out of memory")
    for i in 0..<code_points.length:
        storage[i] = code_points[i]
    return utf32_view_from_span(storage[..<code_points.length])

# mark: Creation ================================================================================

test "short_ascii_literal_is_constexpr":
    # The _utf16 UDL folds short ASCII literals to compile-time constants, both for char and
    # char16_t inputs.
    let from_char = literalutf16_u8("foo".bytes.data, 3)
    let from_char16 = t16_lit("bar")
    assert(t16s_eq_str(&from_char, "foo"))
    assert(t16s_eq_str(&from_char16, "bar"))
    assert(utf16_string_base_length_in_code_units(&from_char.utf16_string_base) == 3)
    assert(utf16_string_base_length_in_code_units(&from_char16.utf16_string_base) == 3)

test "empty_string":
    let string = utf16_string_new()
    t16s_expect(&string, true, false, true, 0, 0)
    let ascii = utf16_string_base_ascii_view(&string.utf16_string_base)
    assert(string_view_eq_string_view(&ascii, StringView()))

test "from_utf8":
    let string1 = utf16_string_from_utf8_string_view(sv("hello!"))
    t16s_expect(&string1, true, false, true, 6, 6)
    assert(t16s_ascii_view_is(&string1, "hello!"))

    let string2 = utf16_string_from_utf8_string_view(sv("hello there!"))
    t16s_expect(&string2, true, true, false, 12, 12)
    assert(t16s_ascii_view_is(&string2, "hello there!"))

    let string3 = utf16_string_from_utf8_string_view(sv("😀"))
    t16s_expect(&string3, false, false, false, 2, 1)
    assert(t16_eq(t16s_view(&string3), t16("😀")))

    let string4 = utf16_string_from_utf8_string_view(sv("hello 😀 there!"))
    t16s_expect(&string4, false, false, false, 15, 14)
    assert(t16_eq(t16s_view(&string4), t16("hello 😀 there!")))

    let string5 = utf16_string_from_utf8_string_view(string_view_from_bytes(b"hello \xed\xa0\x80!"))
    t16s_expect(&string5, false, false, false, 8, 8)
    assert(t16_eq(t16s_view(&string5), t16u([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0xd800, 0x21])))

    let string6 = utf16_string_from_utf8_string_view(string_view_from_bytes(b"hello \xed\xb0\x80!"))
    t16s_expect(&string6, false, false, false, 8, 8)
    assert(t16_eq(t16s_view(&string6), t16u([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0xdc00, 0x21])))

test "from_utf8_with_replacement_character":
    # U+110000
    let string1 = utf16_string_from_utf8_with_replacement_character(string_view_from_bytes(b"long string \xf4\x8f\xbf\xc0"), Utf16StringWithBomHandling.no)
    assert(t16s_eq(&string1, t16("long string \u{fffd}\u{fffd}\u{fffd}\u{fffd}")))

    let string3 = utf16_string_from_utf8_with_replacement_character(sv("A valid string!"), Utf16StringWithBomHandling.no)
    assert(t16s_eq_str(&string3, "A valid string!"))

    let string4 = utf16_string_from_utf8_with_replacement_character(sv(""), Utf16StringWithBomHandling.no)
    assert(t16s_eq_str(&string4, ""))

    let string5 = utf16_string_from_utf8_with_replacement_character(string_view_from_bytes(b"\xef\xbb\xbfWHF!"), Utf16StringWithBomHandling.yes)
    assert(t16s_eq_str(&string5, "WHF!"))

    let string6 = utf16_string_from_utf8_with_replacement_character(string_view_from_bytes(b"\xef\xbb\xbfWHF!"), Utf16StringWithBomHandling.no)
    assert(t16s_eq(&string6, t16("\u{feff}WHF!")))

    # U+D800
    let string7 = utf16_string_from_utf8_with_replacement_character(string_view_from_bytes(b"\xed\xa0\x80WHF!"))
    assert(t16s_eq(&string7, t16("\u{fffd}WHF!")))

test "from_utf16":
    let string1 = utf16_string_from_utf16(t16("hello!"))
    t16s_expect(&string1, true, false, true, 6, 6)
    assert(t16s_ascii_view_is(&string1, "hello!"))

    let string2 = utf16_string_from_utf16(t16("hello there!"))
    t16s_expect(&string2, true, true, false, 12, 12)
    assert(t16s_ascii_view_is(&string2, "hello there!"))

    let string3 = utf16_string_from_utf16(t16("😀"))
    t16s_expect(&string3, false, false, false, 2, 1)
    assert(t16_eq(t16s_view(&string3), t16("😀")))

    let string4 = utf16_string_from_utf16(t16("hello 😀 there!"))
    t16s_expect(&string4, false, false, false, 15, 14)
    assert(t16_eq(t16s_view(&string4), t16("hello 😀 there!")))

    let high = t16u([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0xd800, 0x21])
    let string5 = utf16_string_from_utf16(high)
    t16s_expect(&string5, false, false, false, 8, 8)
    assert(t16_eq(t16s_view(&string5), high))

    let low = t16u([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0xdc00, 0x21])
    let string6 = utf16_string_from_utf16(low)
    t16s_expect(&string6, false, false, false, 8, 8)
    assert(t16_eq(t16s_view(&string6), low))

test "from_utf32":
    let string1 = utf16_string_from_utf32(t16_utf32([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x21]))
    t16s_expect(&string1, true, false, true, 6, 6)
    assert(t16s_ascii_view_is(&string1, "hello!"))

    let string2 = utf16_string_from_utf32(t16_utf32([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x74, 0x68, 0x65, 0x72, 0x65, 0x21]))
    t16s_expect(&string2, true, true, false, 12, 12)
    assert(t16s_ascii_view_is(&string2, "hello there!"))

    let string3 = utf16_string_from_utf32(t16_utf32([0x1f600]))
    t16s_expect(&string3, false, false, false, 2, 1)
    assert(t16_eq(t16s_view(&string3), t16("😀")))

    let string4 = utf16_string_from_utf32(t16_utf32([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x1f600, 0x20, 0x74, 0x68, 0x65, 0x72, 0x65, 0x21]))
    t16s_expect(&string4, false, false, false, 15, 14)
    assert(t16_eq(t16s_view(&string4), t16("hello 😀 there!")))

    let string5 = utf16_string_from_utf32(t16_utf32([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0xd800, 0x21]))
    t16s_expect(&string5, false, false, false, 8, 8)
    assert(t16_eq(t16s_view(&string5), t16u([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0xd800, 0x21])))

    let string6 = utf16_string_from_utf32(t16_utf32([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0xdc00, 0x21]))
    t16s_expect(&string6, false, false, false, 8, 8)
    assert(t16_eq(t16s_view(&string6), t16u([0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0xdc00, 0x21])))

## Collects the code units code_point_to_utf16 produces.
struct T16CodeUnits: Utf16CodeUnitCallback:
    var units: u16[2]
    var count: usize

    func call(code_unit: u16):
        self.units[self.count] = code_unit
        self.count += 1

test "from_code_point":
    with memory.heap:
        var code_point: u32 = 0

        while code_point < unicode_utils_first_supplementary_plane_code_point:
            let string = utf16_string_from_code_point(code_point)
            assert(utf16_string_base_length_in_code_units(&string.utf16_string_base) == 1)
            assert(utf16_string_base_length_in_code_points(&string.utf16_string_base) == 1)
            assert(utf16_string_base_code_point_at(&string.utf16_string_base, 0) == code_point)
            assert((u32)utf16_string_base_code_unit_at(&string.utf16_string_base, 0) == code_point)
            code_point += 1

        while code_point < unicode_utils_first_supplementary_plane_code_point + 10_000:
            let string = utf16_string_from_code_point(code_point)
            assert(utf16_string_base_length_in_code_units(&string.utf16_string_base) == 2)
            assert(utf16_string_base_length_in_code_points(&string.utf16_string_base) == 1)
            assert(utf16_string_base_code_point_at(&string.utf16_string_base, 0) == code_point)

            var units = T16CodeUnits()
            _ = unicode_utils_code_point_to_utf16(code_point, &units)
            assert(units.count == 2)
            for i in 0..<units.count:
                assert(utf16_string_base_code_unit_at(&string.utf16_string_base, i) == units.units[i])
            code_point += 1

test "formatted":
    # Utf16String::formatted("{}", 42) and formatted("whf {} {} {}!", ...) are variadic
    # templates the port does not have.
    let string = utf16_string_number_i64(42)
    t16s_expect(&string, true, false, true, 2, 2)
    assert(t16s_eq(&string, t16("42")))

    let segments1: Utf16View[6] = [t16("abcdefghijklmnopqrstuvwxyz"), t16("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), t16("abcdefghijklmnopqrstuvwxyz"), t16("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), t16("abcdefghijklmnopqrstuvwxyz"), t16("ABCDEFGHIJKLMNOPQRSTUVWXYZ")]
    let joined1 = utf16_string_join_utf16_view(t16("--"), segments1)
    t16s_expect(&joined1, true, true, false, 166, 166)
    assert(t16s_eq(&joined1, t16("abcdefghijklmnopqrstuvwxyz--ABCDEFGHIJKLMNOPQRSTUVWXYZ--abcdefghijklmnopqrstuvwxyz--ABCDEFGHIJKLMNOPQRSTUVWXYZ--abcdefghijklmnopqrstuvwxyz--ABCDEFGHIJKLMNOPQRSTUVWXYZ")))

    let segments2: Utf16View[9] = [t16("abcdefghijklmnopqrstuvwxyz"), t16("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), t16u([0xd83d, 0xde00]), t16("abcdefghijklmnopqrstuvwxyz"), t16u([0xd800, 0xdc00]), t16("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), t16("🍕"), t16("abcdefghijklmnopqrstuvwxyz"), t16("ABCDEFGHIJKLMNOPQRSTUVWXYZ")]
    let joined2 = utf16_string_join_utf16_view(t16("--"), segments2)
    t16s_expect(&joined2, false, false, false, 178, 175)
    assert(t16s_eq(&joined2, t16("abcdefghijklmnopqrstuvwxyz--ABCDEFGHIJKLMNOPQRSTUVWXYZ--😀--abcdefghijklmnopqrstuvwxyz--𐀀--ABCDEFGHIJKLMNOPQRSTUVWXYZ--🍕--abcdefghijklmnopqrstuvwxyz--ABCDEFGHIJKLMNOPQRSTUVWXYZ")))

## `Utf16String::repeated(code_point, count)` has `length` code units and equals `expected`.
func t16_repeated_is(code_point: u32, count: usize, length: usize, expected: str) -> bool:
    let string = utf16_string_repeated(code_point, count)
    return utf16_string_base_length_in_code_units(&string.utf16_string_base) == length and t16s_eq(&string, t16(expected))

test "repeated":
    let empty1 = utf16_string_repeated(0x61, 0)
    assert(utf16_string_base_is_empty(&empty1.utf16_string_base))

    let empty2 = utf16_string_repeated(0x03c9, 0)
    assert(utf16_string_base_is_empty(&empty2.utf16_string_base))
    let empty3 = utf16_string_repeated(0x10300, 0)
    assert(utf16_string_base_is_empty(&empty3.utf16_string_base))

    assert(t16_repeated_is(0x61, 1, 1, "a"))
    assert(t16_repeated_is(0x03c9, 1, 1, "ω"))
    assert(t16_repeated_is(0x10300, 1, 2, "𐌀"))

    assert(t16_repeated_is(0x61, 3, 3, "aaa"))
    assert(t16_repeated_is(0x03c9, 3, 3, "ωωω"))
    assert(t16_repeated_is(0x10300, 3, 6, "𐌀𐌀𐌀"))

    assert(t16_repeated_is(0x61, 10, 10, "aaaaaaaaaa"))
    assert(t16_repeated_is(0x03c9, 10, 10, "ωωωωωωωωωω"))
    assert(t16_repeated_is(0x10300, 10, 20, "𐌀𐌀𐌀𐌀𐌀𐌀𐌀𐌀𐌀𐌀"))

test "from_string_builder":
    var builder = string_builder_new_mode(StringBuilderMode.utf16)
    string_builder_append_code_point(&builder, 0x61)
    string_builder_append_code_point(&builder, 0x62)
    string_builder_append_code_point(&builder, 0x1f600)
    string_builder_append_code_point(&builder, 0x10000)
    string_builder_append_code_point(&builder, 0x1f355)
    string_builder_append_code_point(&builder, 0x63)
    string_builder_append_code_point(&builder, 0x64)

    let string = string_builder_to_utf16_string(&builder)
    assert(utf16_string_base_length_in_code_units(&string.utf16_string_base) == 10)
    assert(utf16_string_base_length_in_code_points(&string.utf16_string_base) == 7)
    assert(t16s_eq_str(&string, "ab😀𐀀🍕cd"))

## `Utf16String::from_ipc_stream(FixedMemoryStream { bytes }, length, is_ascii)`.
func t16_from_ipc_stream(bytes: const u8[], length_in_code_units: usize, is_ascii_: bool) -> Utf16String!:
    var stream: FixedMemoryStream = ---
    fixed_memory_stream_construct(&stream, bytes)
    return try utf16_string_from_ipc_stream((Stream*)&stream, length_in_code_units, is_ascii_)

## Whether from_ipc_stream fails.
func t16_from_ipc_stream_fails(bytes: const u8[], length_in_code_units: usize, is_ascii_: bool) -> bool:
    var failed = false
    _ = t16_from_ipc_stream(bytes, length_in_code_units, is_ascii_) catch:
        failed = true
        recover utf16_string_new()
    return failed

## The bytes of a UTF-16 builder holding `view` (`builder.to_byte_buffer()`).
func t16_utf16_bytes(view: Utf16View) -> ByteBuffer:
    var builder = string_builder_new_mode(StringBuilderMode.utf16)
    string_builder_append_utf16_view(&builder, view)
    return string_builder_to_byte_buffer(&builder) else trap("MUST: to_byte_buffer")

test "from_ipc_stream":
    # FixedMemoryStream belongs to region r01.
    let string1 = t16_from_ipc_stream("abc".bytes, 3, true) else trap("from_ipc_stream")
    t16s_expect(&string1, true, false, true, 3, 3)
    assert(t16s_eq_str(&string1, "abc"))

    let string2 = t16_from_ipc_stream("abcdefghijklmnopqrstuvwxyz".bytes, 26, true) else trap("from_ipc_stream")
    t16s_expect(&string2, true, true, false, 26, 26)
    assert(t16s_eq_str(&string2, "abcdefghijklmnopqrstuvwxyz"))

    let data3 = t16("hello 😀 there!")
    let buffer3 = t16_utf16_bytes(data3)
    let string3 = t16_from_ipc_stream(byte_buffer_bytes_const(&buffer3), 15, false) else trap("from_ipc_stream")
    t16s_expect(&string3, false, false, false, 15, 14)
    assert(t16s_eq(&string3, data3))

    assert(t16_from_ipc_stream_fails("abc".bytes, 4, true))

    let buffer5 = t16_utf16_bytes(t16("😀"))
    assert(t16_from_ipc_stream_fails(byte_buffer_bytes_const(&buffer5), 2, true))

    let buffer6 = t16_utf16_bytes(data3)
    assert(t16_from_ipc_stream_fails(byte_buffer_bytes_const(&buffer6), 15, true))