Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.0

0.6.0 / src / ak / tests_text_support.lucb

#==============================================================================================
#
#   tests_text_support - tests of the region's code that has no donor tests
#
#   DESCRIPTION:
#       Focused tests pinned to the C++ behaviour for simdutf (results checked against the
#       donor build's libsimdutf 7.4.0), MemMem, UnicodeUtils, StringBuilder, StringNumber and
#       Utf32View.
#
#==============================================================================================

# mark: simdutf =================================================================================

## validate_utf8_with_errors, count_utf8 and utf16_length_from_utf8 of `bytes`.
func tc_check_utf8(bytes: const u8[], expected_error: SimdutfErrorCode, count: usize, count_utf8: usize, utf16_length: usize):
    let result = simdutf_validate_utf8_with_errors(bytes)
    assert(result.error == expected_error)
    assert(result.count == count)
    assert(simdutf_count_utf8(bytes) == count_utf8)
    assert(simdutf_utf16_length_from_utf8(bytes) == utf16_length)

test "simdutf: UTF-8 validation errors and lengths":
    tc_check_utf8(b"hello", SimdutfErrorCode.success, 5, 5, 5)
    tc_check_utf8(b"h\xc3\xa9\xe2\x82\xac\xf0\x9f\x98\x80", SimdutfErrorCode.success, 10, 4, 5)
    tc_check_utf8(b"ab\xff", SimdutfErrorCode.header_bits, 2, 3, 4)
    tc_check_utf8(b"ab\x80", SimdutfErrorCode.too_long, 2, 2, 2)
    tc_check_utf8(b"ab\xe2\x82", SimdutfErrorCode.too_short, 2, 3, 3)
    tc_check_utf8(b"a\xc0\xaf", SimdutfErrorCode.overlong, 1, 2, 2)
    tc_check_utf8(b"a\xed\xa0\x80z", SimdutfErrorCode.surrogate, 1, 3, 3)
    tc_check_utf8(b"a\xf4\x90\x80\x80", SimdutfErrorCode.too_large, 1, 2, 3)
    assert(simdutf_validate_ascii(b"plain"))
    assert(not simdutf_validate_ascii(b"caf\xc3\xa9"))

test "simdutf: UTF-8 to UTF-16":
    var output: u16[8]
    let written = simdutf_convert_utf8_to_utf16(b"h\xc3\xa9\xe2\x82\xac\xf0\x9f\x98\x80", output)
    assert(written == 5)
    let expected: u16[5] = [0x0068, 0x00e9, 0x20ac, 0xd83d, 0xde00]
    for i in 0..<expected.length:
        assert(output[i] == expected[i])
    assert(simdutf_convert_utf8_to_utf16(b"a\xff", output) == 0)

test "simdutf: UTF-16 validation, lengths and well-formed copies":
    let units: u16[7] = [0x41, 0xd83d, 0xde00, 0xe9, 0xdc00, 0x42, 0xd800]
    let result = simdutf_validate_utf16_with_errors(units)
    assert(result.error == SimdutfErrorCode.surrogate)
    assert(result.count == 4)
    assert(not simdutf_validate_utf16(units))
    assert(simdutf_validate_utf16(units[..<4]))
    assert(simdutf_count_utf16(units[..<4]) == 3)
    assert(simdutf_utf8_length_from_utf16(units) == 12)
    assert(simdutf_validate_utf16_as_ascii(units[..<1]))
    assert(not simdutf_validate_utf16_as_ascii(units))

    var well_formed: u16[7]
    simdutf_to_well_formed_utf16(units, well_formed)
    let expected: u16[7] = [0x0041, 0xd83d, 0xde00, 0x00e9, 0xfffd, 0x0042, 0xfffd]
    for i in 0..<expected.length:
        assert(well_formed[i] == expected[i])

test "simdutf: UTF-16 to UTF-8":
    let units: u16[7] = [0x41, 0xd83d, 0xde00, 0xe9, 0xdc00, 0x42, 0xd800]
    var output: u8[32]
    let written = simdutf_convert_utf16_to_utf8(units[..<4], output)
    assert(written == 7)
    assert(output[..<written] == b"\x41\xf0\x9f\x98\x80\xc3\xa9")

    let (result, partial) = simdutf_convert_utf16_to_utf8_with_errors(units, output)
    assert(result.error == SimdutfErrorCode.surrogate)
    assert(result.count == 4)
    assert(partial == 7)
    assert(simdutf_convert_utf16_to_utf8(units, output) == 0)

test "simdutf: UTF-16 bytes in either order":
    let big_endian = b"\x00\x41\xd8\x3d\xde\x00\xd8\x00"
    assert(simdutf_validate_utf16_bytes(big_endian[..<6], true))
    assert(not simdutf_validate_utf16_bytes(big_endian, true))
    assert(simdutf_validate_utf16_bytes(big_endian[..<6], false))
    assert(simdutf_utf8_length_from_utf16_bytes(big_endian[..<6], true) == 5)
    let well_formed = simdutf_to_well_formed_utf16_bytes(big_endian, true)
    assert(well_formed.length == 4)
    assert(well_formed[0] == 0x41 and well_formed[1] == 0xd83d and well_formed[2] == 0xde00 and well_formed[3] == 0xfffd)
    var output: u8[16]
    assert(simdutf_convert_utf16_bytes_to_utf8(big_endian[..<6], true, output) == 5)
    assert(output[..<5] == b"\x41\xf0\x9f\x98\x80")

test "simdutf: UTF-32":
    let code_points: u32[4] = [0x41, 0xe9, 0x20ac, 0x1f600]
    assert(simdutf_validate_utf32(code_points))
    assert(simdutf_utf16_length_from_utf32(code_points) == 5)
    var utf8: u8[16]
    let utf8_written = simdutf_convert_utf32_to_utf8(code_points, utf8)
    assert(utf8_written == 10)
    assert(utf8[..<utf8_written] == b"\x41\xc3\xa9\xe2\x82\xac\xf0\x9f\x98\x80")
    var utf16: u16[8]
    assert(simdutf_convert_utf32_to_utf16(code_points, utf16) == 5)
    assert(utf16[3] == 0xd83d and utf16[4] == 0xde00)

    let surrogate: u32[2] = [0x41, 0xd800]
    assert(not simdutf_validate_utf32(surrogate))
    assert(simdutf_convert_utf32_to_utf8(surrogate, utf8) == 0)
    let too_large: u32[1] = [0x110000]
    assert(not simdutf_validate_utf32(too_large))
    assert(simdutf_convert_utf32_to_utf8(too_large, utf8) == 0)

test "simdutf: find":
    assert(simdutf_find(b"abcabc", 'c') == 2)
    assert(simdutf_find(b"abcabc", 'x') == none)
    let units: u16[3] = [1, 2, 2]
    assert(simdutf_find_u16(units, 2) == 1)
    assert(simdutf_find_u16(units, 3) == none)

# mark: MemMem ==================================================================================

test "memmem: short needles (bitap)":
    assert(memmem_optional(b"hello", b"") == 0)
    assert(memmem_optional(b"he", b"hello") == none)
    assert(memmem_optional(b"hello", b"hello") == 0)
    assert(memmem_optional(b"hello", b"hellx") == none)
    assert(memmem_optional(b"hello world", b"o") == 4)
    assert(memmem_optional(b"hello world", b"world") == 6)
    assert(memmem_optional(b"hello world", b"worlds") == none)
    assert(memmem_optional(b"aaaab", b"aab") == 2)

test "memmem: long needles (KMP)":
    let needle = b"abcdefghijklmnopqrstuvwxyz0123456789"
    assert(memmem_optional(b"--abcdefghijklmnopqrstuvwxyz0123456789--", needle) == 2)
    assert(memmem_optional(b"--abcdefghijklmnopqrstuvwxyz012345678--", needle) == none)
    # A needle with a repeated prefix exercises the KMP table's fallbacks.
    let repeated = b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"
    assert(repeated.length == 33)
    assert(memmem_optional(b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab", repeated) == 12)
    assert(memmem_optional(b"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", repeated) == none)

# mark: UnicodeUtils ============================================================================

## A UTF-16 callback that records the code units.
struct TcCodeUnits: Utf16CodeUnitCallback:
    var units: u16[2]
    var count: usize

    func call(code_unit: u16):
        self.units[self.count] = code_unit
        self.count += 1

test "unicode utils: UTF-8 encoding":
    assert(unicode_utils_bytes_to_store_code_point_in_utf8(0x7f) == 1)
    assert(unicode_utils_bytes_to_store_code_point_in_utf8(0x80) == 2)
    assert(unicode_utils_bytes_to_store_code_point_in_utf8(0x7ff) == 2)
    assert(unicode_utils_bytes_to_store_code_point_in_utf8(0x800) == 3)
    assert(unicode_utils_bytes_to_store_code_point_in_utf8(0xffff) == 3)
    assert(unicode_utils_bytes_to_store_code_point_in_utf8(0x10000) == 4)
    assert(unicode_utils_bytes_to_store_code_point_in_utf8(0x10ffff) == 4)
    assert(unicode_utils_bytes_to_store_code_point_in_utf8(0x110000) == 0)

    var utf8 = CodePointAsUtf8()
    assert(unicode_utils_code_point_to_utf8(0x20ac, &utf8) == 3)
    assert(utf8.bytes[..<utf8.length] == b"\xe2\x82\xac")
    var too_large = CodePointAsUtf8()
    assert(unicode_utils_code_point_to_utf8(0x110000, &too_large) == -1)
    assert(too_large.length == 0)

test "unicode utils: UTF-16 encoding":
    assert(unicode_utils_code_unit_length_for_code_point(0xffff) == 1)
    assert(unicode_utils_code_unit_length_for_code_point(0x10000) == 2)
    assert(unicode_utils_is_utf16_high_surrogate(0xd800) and unicode_utils_is_utf16_high_surrogate(0xdbff))
    assert(not unicode_utils_is_utf16_high_surrogate(0xdc00))
    assert(unicode_utils_is_utf16_low_surrogate(0xdc00) and unicode_utils_is_utf16_low_surrogate(0xdfff))
    assert(not unicode_utils_is_utf16_low_surrogate(0xdbff))
    assert(unicode_utils_decode_utf16_surrogate_pair(0xd83d, 0xde00) == 0x1f600)

    var units = TcCodeUnits()
    assert(unicode_utils_code_point_to_utf16(0x1f600, &units) == 2)
    assert(units.units[0] == 0xd83d and units.units[1] == 0xde00)
    var single = TcCodeUnits()
    assert(unicode_utils_code_point_to_utf16(0xe9, &single) == 1)
    assert(single.units[0] == 0xe9)

    let mixed: u16[5] = [0x41, 0x7ff, 0x800, 0xd83d, 0xde00]
    assert(unicode_utils_maximum_utf8_length_from_utf16(mixed) == 1 + 2 + 3 + 3 + 3)

# mark: StringBuilder ===========================================================================

## Whether a UTF-16 view holds exactly `expected`.
func tc_utf16_is(view: const Utf16View*, expected: const u16[]) -> bool:
    if utf16_view_length_in_code_units(view) != expected.length:
        return false
    for i in 0..<expected.length:
        if utf16_view_code_unit_at(view, i) != expected[i]:
            return false
    return true

test "string builder: UTF-8 code points":
    var builder = string_builder_new()
    string_builder_append_code_point(&builder, (u32)'a')
    string_builder_append_code_point(&builder, 0xe9)
    string_builder_append_code_point(&builder, 0x20ac)
    string_builder_append_code_point(&builder, 0x1f600)
    string_builder_append_code_point(&builder, 0x110000)
    let view = string_builder_string_view(&builder)
    assert(string_view_bytes(&view) == b"a\xc3\xa9\xe2\x82\xac\xf0\x9f\x98\x80\xef\xbf\xbd")
    assert(string_builder_length(&builder) == 13)

    try string_builder_try_append_code_point(&builder, 0x42)
    string_builder_trim(&builder, 4)
    let trimmed = string_builder_string_view(&builder)
    assert(string_view_bytes(&trimmed) == b"a\xc3\xa9\xe2\x82\xac\xf0\x9f\x98\x80")

    string_builder_clear(&builder)
    assert(string_builder_is_empty(&builder))

test "string builder: UTF-16 mode switches from ASCII storage":
    var builder = string_builder_new_mode(StringBuilderMode.utf16)
    string_builder_append(&builder, "abc")
    let ascii_view = string_builder_utf16_string_view(&builder)
    assert(utf16_view_has_ascii_storage(&ascii_view))
    assert(utf16_view_length_in_code_units(&ascii_view) == 3)

    string_builder_append_code_point(&builder, 0xe9)
    string_builder_append_code_point(&builder, 0x1f600)
    string_builder_append_u8(&builder, 'z')
    let utf16_view = string_builder_utf16_string_view(&builder)
    assert(not utf16_view_has_ascii_storage(&utf16_view))
    let expected: u16[7] = [0x61, 0x62, 0x63, 0xe9, 0xd83d, 0xde00, 0x7a]
    assert(tc_utf16_is(&utf16_view, expected))
    assert(string_builder_length(&builder) == 14)

    string_builder_trim(&builder, 1)
    let trimmed = string_builder_utf16_string_view(&builder)
    assert(tc_utf16_is(&trimmed, expected[..<6]))

    let string = string_builder_to_utf16_string(&builder)
    assert(utf16_string_base_has_long_utf16_storage(&string.utf16_string_base))
    assert(utf16_string_base_length_in_code_units(&string.utf16_string_base) == 6)
    assert(utf16_string_base_length_in_code_points(&string.utf16_string_base) == 5)

test "string builder: UTF-16 views appended in UTF-8 mode":
    var builder = string_builder_new()
    let units: u16[5] = [0x41, 0xd83d, 0xde00, 0xdc00, 0x42]
    string_builder_append_utf16_view(&builder, utf16_view_from_utf16(units))
    let view = string_builder_string_view(&builder)
    # The lonely low surrogate is encoded as three bytes (WTF-8).
    assert(string_view_bytes(&view) == b"\x41\xf0\x9f\x98\x80\xed\xb0\x80\x42")

test "string builder: append_escaped_for_json":
    var builder = string_builder_new()
    string_builder_append_escaped_for_json(&builder, string_view_from_bytes(b"\"\\\x08\n\t\x01\x1fx"))
    let view = string_builder_string_view(&builder)
    assert(tc_view_is(view, "\\\"\\\\\\b\\n\\t\\u0001\\u001fx"))

test "string builder: append_repeated and append_as_lowercase":
    var builder = string_builder_new()
    string_builder_append_repeated_u8(&builder, '-', 3)
    string_builder_append_repeated_string_view(&builder, sv("ab"), 2)
    string_builder_append_as_lowercase(&builder, 'Q')
    string_builder_append_as_lowercase(&builder, '1')
    let view = string_builder_string_view(&builder)
    assert(tc_view_is(view, "---ababq1"))
    let string = string_builder_to_string(&builder) else trap("MUST")
    assert(string_eq_string_view(&string, sv("---ababq1")))

# mark: StringNumber ============================================================================

test "string number: extremes":
    let minimum = string_number_i64(-9223372036854775808)
    assert(string_eq_string_view(&minimum, sv("-9223372036854775808")))
    let maximum = string_number_i64(9223372036854775807)
    assert(string_eq_string_view(&maximum, sv("9223372036854775807")))
    let zero = string_number_i64(0)
    assert(string_eq_string_view(&zero, sv("0")))
    let negative = string_number_i64(-42)
    assert(string_eq_string_view(&negative, sv("-42")))
    let unsigned_maximum = string_number_u64(18446744073709551615)
    assert(string_eq_string_view(&unsigned_maximum, sv("18446744073709551615")))

    let utf16 = utf16_string_number_i64(-123456789)
    assert(utf16_string_base_eq_string_view(&utf16.utf16_string_base, sv("-123456789")))
    let utf16_unsigned = utf16_string_number_u64(7)
    assert(utf16_string_base_eq_string_view(&utf16_unsigned.utf16_string_base, sv("7")))

    let byte_string = byte_string_number_i64(-5)
    assert(tc_bs_is(&byte_string, "-5"))
    let byte_string_unsigned = byte_string_number_u64(18446744073709551615)
    assert(tc_bs_is(&byte_string_unsigned, "18446744073709551615"))

# mark: Utf32View ===============================================================================

test "utf32 view":
    let code_points: u32[4] = [0x41, 0xe9, 0x20ac, 0x1f600]
    let view = utf32_view_from_span(code_points)
    assert(utf32_view_length(&view) == 4)
    assert(not utf32_view_is_empty(&view))
    assert(not utf32_view_is_null(&view))
    assert(not utf32_view_is_ascii(&view))
    assert(utf32_view_at_const(&view, 3) == 0x1f600)

    let tail = utf32_view_substring_view(&view, 2)
    assert(utf32_view_length(&tail) == 2)
    assert(utf32_view_at_const_2(&tail, 0) == 0x20ac)
    let head = utf32_view_substring_view_usize(&view, 0, 1)
    assert(utf32_view_is_ascii(&head))

    let same = utf32_view_from_span(code_points)
    assert(utf32_view_eq(&view, &same))
    assert(not utf32_view_eq(&view, &tail))

    var it = utf32_view_begin(&view)
    assert(utf32_code_point_iterator_peek(&it, 3) == 0x1f600)
    assert(utf32_code_point_iterator_peek(&it, 4) == none)
    _ = utf32_code_point_iterator_inc(&it)
    assert(utf32_view_iterator_offset(&view, &it) == 1)
    assert(utf32_code_point_iterator_deref(&it) == 0xe9)
    let begin = utf32_view_begin(&view)
    assert(utf32_code_point_iterator_sub(&it, &begin) == 1)
    assert(utf32_code_point_iterator_code_point_length_in_bytes(&it) == 4)

    var count: usize = 0
    var walker = utf32_view_begin(&view)
    let end = utf32_view_end(&view)
    while not utf32_code_point_iterator_eq(&walker, &end):
        count += 1
        _ = utf32_code_point_iterator_inc(&walker)
    assert(count == 4)
    assert(utf32_code_point_iterator_done(&walker))

    let empty = Utf32View()
    assert(utf32_view_is_null(&empty))
    assert(utf32_view_is_empty(&empty))