dymokomi/luce-browser-foundation
Source of release 0.6.0
0.6.0 / src / ak / utf16_string_data.lucb
#==============================================================================================
#
# utf16_string_data - Utf16StringData: the heap block behind a long Utf16String
#
# DESCRIPTION:
# Port of AK/Utf16StringData.h (lines 1-160) and AK/Utf16StringData.cpp (lines 1-215).
#
# The code units follow the header at offset_of_string_storage(): ASCII bytes while the
# string is all ASCII, otherwise host-endian UTF-16 (the storage kind is the top bit of
# m_length_in_code_units). Blocks come from the current allocator (the managed heap once
# region gc lands); there is no reference count and no destructor (DESIGN.md ยง3.3): the
# Utf16FlyString table entry is removed by did_destroy_utf16_fly_string_data, which the
# collector's finalizer will call.
#
#==============================================================================================
# mark: Creation ================================================================================
## Ported from AK/Utf16StringData.cpp:15 VERIFY_UTF16_LENGTH: due to internal optimizations,
## the maximum string length is 2**63 - 1.
func verify_utf16_length(length: usize):
assert(length >> utf16_flag == 0)
## Ported from AK/Utf16StringData.cpp:67 AK::Detail::Utf16StringData::from_utf8.
pub func utf16_string_data_from_utf8(utf8_string: StringView, allow_ascii_storage: Utf16StringDataAllowAsciiStorage) -> Utf16StringData*:
if allow_ascii_storage == Utf16StringDataAllowAsciiStorage.yes and string_view_is_ascii(&utf8_string):
verify_utf16_length(string_view_length(&utf8_string))
let string = utf16_string_data_create_uninitialized(Utf16StringDataStorageType.ascii, string_view_length(&utf8_string))
copy_bytes(utf16_string_data_ascii_storage(string), string_view_bytes(&utf8_string))
return string
let view = Utf8View(m_string = utf8_string)
if utf8_view_validate_allow_lonely_surrogates(&view, AllowLonelySurrogates.no):
let code_unit_length = simdutf_utf16_length_from_utf8(string_view_bytes(&utf8_string))
verify_utf16_length(code_unit_length)
let string = utf16_string_data_create_uninitialized(Utf16StringDataStorageType.utf16, code_unit_length)
let result = simdutf_convert_utf8_to_utf16(string_view_bytes(&utf8_string), utf16_string_data_utf16_storage(string))
assert(result == code_unit_length)
return string
return utf16_string_data_create_from_code_point_iterable_utf8_view(view)
## Ported from AK/Utf16StringData.cpp:59 AK::Detail::Utf16StringData::from_ascii.
pub func utf16_string_data_from_ascii(ascii_string: const u8[]) -> Utf16StringData*:
verify_utf16_length(ascii_string.length)
let string = utf16_string_data_create_uninitialized(Utf16StringDataStorageType.ascii, ascii_string.length)
copy_bytes(utf16_string_data_ascii_storage(string), ascii_string)
return string
## Ported from AK/Utf16StringData.cpp:91 AK::Detail::Utf16StringData::from_utf16.
pub func utf16_string_data_from_utf16(utf16_string: Utf16View) -> Utf16StringData*:
verify_utf16_length(utf16_view_length_in_code_units(&utf16_string))
if utf16_view_has_ascii_storage(&utf16_string):
let string = utf16_string_data_create_uninitialized(Utf16StringDataStorageType.ascii, utf16_view_length_in_code_units(&utf16_string))
copy_bytes(utf16_string_data_ascii_storage(string), utf16_view_ascii_span(&utf16_string))
return string
if utf16_view_is_ascii(&utf16_string):
let string = utf16_string_data_create_uninitialized(Utf16StringDataStorageType.ascii, utf16_view_length_in_code_units(&utf16_string))
let result = simdutf_convert_utf16_to_utf8(utf16_view_utf16_span(&utf16_string), utf16_string_data_ascii_storage(string))
assert(result == utf16_view_length_in_code_units(&utf16_string))
return string
let string = utf16_string_data_create_uninitialized(Utf16StringDataStorageType.utf16, utf16_view_length_in_code_units(&utf16_string))
let storage = utf16_string_data_utf16_storage(string)
let source = utf16_view_utf16_span(&utf16_string)
for i in 0..<source.length:
storage[i] = source[i]
string.m_length_in_code_points = utf16_string.m_length_in_code_points
return string
## Ported from AK/Utf16StringData.cpp:114 AK::Detail::Utf16StringData::from_utf32.
pub func utf16_string_data_from_utf32(utf32_string: Utf32View) -> Utf16StringData*:
let utf32_data = utf32_view_span(&utf32_string)
let utf32_length = utf32_view_length(&utf32_string)
if utf32_view_is_ascii(&utf32_string):
verify_utf16_length(utf32_length)
let string = utf16_string_data_create_uninitialized(Utf16StringDataStorageType.ascii, utf32_length)
let result = simdutf_convert_utf32_to_utf8(utf32_data, utf16_string_data_ascii_storage(string))
assert(result == utf32_length)
return string
if simdutf_validate_utf32(utf32_data):
let code_unit_length = simdutf_utf16_length_from_utf32(utf32_data)
verify_utf16_length(code_unit_length)
let string = utf16_string_data_create_uninitialized(Utf16StringDataStorageType.utf16, code_unit_length)
string.m_length_in_code_points = utf32_length
let result = simdutf_convert_utf32_to_utf16(utf32_data, utf16_string_data_utf16_storage(string))
assert(result == code_unit_length)
return string
return utf16_string_data_create_from_code_point_iterable_utf32_view(utf32_string)
## Ported from AK/Utf16StringData.cpp:144 AK::Detail::Utf16StringData::from_string_builder. The
## C++ adopts the builder's heap buffer when it has one; the port always copies (see
## string_data_create_from_string_builder) and clears the builder as a leak would.
pub func utf16_string_data_from_string_builder(builder: StringBuilder*) -> Utf16StringData*:
let view = string_builder_utf16_string_view(builder)
let code_unit_length = utf16_view_length_in_code_units(&view)
verify_utf16_length(code_unit_length)
let storage_type = Utf16StringDataStorageType.ascii if utf16_view_has_ascii_storage(&view) else Utf16StringDataStorageType.utf16
let string = utf16_string_data_create_uninitialized(storage_type, code_unit_length)
if utf16_view_has_ascii_storage(&view):
copy_bytes(utf16_string_data_ascii_storage(string), utf16_view_ascii_span(&view))
else:
let storage = utf16_string_data_utf16_storage(string)
let source = utf16_view_utf16_span(&view)
for i in 0..<source.length:
storage[i] = source[i]
string.m_length_in_code_points = view.m_length_in_code_points
string_builder_clear(builder)
return string
## Ported from AK/Utf16StringData.cpp:171 AK::Detail::Utf16StringData::from_ipc_stream: a string of
## `length_in_code_units` ASCII bytes or host-endian code units read from `stream`.
pub func utf16_string_data_from_ipc_stream(stream: Stream*, length_in_code_units: usize, is_ascii_: bool) -> Utf16StringData*!:
if is_ascii_:
let string = utf16_string_data_create_uninitialized(Utf16StringDataStorageType.ascii, length_in_code_units)
try stream_read_until_filled(stream, utf16_string_data_ascii_storage(string))
let ascii = utf16_string_data_ascii_view(string)
if not string_view_is_ascii(&ascii):
error(ak_error, "Stream contains invalid ASCII data")
return string
let string = utf16_string_data_create_uninitialized(Utf16StringDataStorageType.utf16, length_in_code_units)
let bytes = u8[]((u8*)string + utf16_string_data_offset_of_string_storage(), length_in_code_units * memory.size_of(u16))
try stream_read_until_filled(stream, bytes)
return string
## Ported from AK/Utf16StringData.cpp:193 AK::Detail::Utf16StringData::to_well_formed: a copy with
## every lonely surrogate replaced by U+FFFD.
pub func utf16_string_data_to_well_formed(utf16_string: Utf16View) -> Utf16StringData*:
assert(not utf16_view_has_ascii_storage(&utf16_string))
let string = utf16_string_data_create_uninitialized(Utf16StringDataStorageType.utf16, utf16_view_length_in_code_units(&utf16_string))
simdutf_to_well_formed_utf16(utf16_view_utf16_span(&utf16_string), utf16_string_data_utf16_storage(string))
return string
## Ported from AK/Utf16StringData.h:48 AK::Detail::Utf16StringData::offset_of_string_storage.
pub func utf16_string_data_offset_of_string_storage() -> usize:
return memory.size_of(Utf16StringData)
# mark: Comparison and accessors ================================================================
## Ported from AK/Utf16StringData.h:58 AK::Detail::Utf16StringData::operator==(Utf16StringData const&).
pub func utf16_string_data_eq_utf16_string_data(this: const Utf16StringData*, other: const Utf16StringData*) -> bool:
if utf16_string_data_is_fly_string(this) and utf16_string_data_is_fly_string(other):
return this == other
let view = utf16_string_data_utf16_view(this)
let other_view = utf16_string_data_utf16_view(other)
return utf16_view_eq_utf16_view(&view, &other_view)
## Ported from AK/Utf16StringData.h:65 AK::Detail::Utf16StringData::operator==(Utf16View const&).
pub func utf16_string_data_eq_utf16_view(this: const Utf16StringData*, other: const Utf16View*) -> bool:
let view = utf16_string_data_utf16_view(this)
return utf16_view_eq_utf16_view(&view, other)
## Ported from AK/Utf16StringData.h:70 AK::Detail::Utf16StringData::operator==(StringView const&):
## compares with ASCII text code unit by code unit.
pub func utf16_string_data_eq_string_view(this: const Utf16StringData*, other: StringView) -> bool:
if utf16_string_data_has_ascii_storage(this):
let view = utf16_string_data_ascii_view(this)
return string_view_eq_string_view(&view, other)
let view = utf16_string_data_utf16_view(this)
let other_view = utf16_view_from_ascii(string_view_bytes(&other))
return utf16_view_eq_utf16_view(&view, &other_view)
## Ported from AK/Utf16StringData.h:77 AK::Detail::Utf16StringData::has_ascii_storage.
pub func utf16_string_data_has_ascii_storage(this: const Utf16StringData*) -> bool:
return this.m_length_in_code_units >> utf16_flag == 0
## Ported from AK/Utf16StringData.h:78 AK::Detail::Utf16StringData::has_utf16_storage.
pub func utf16_string_data_has_utf16_storage(this: const Utf16StringData*) -> bool:
return this.m_length_in_code_units >> utf16_flag != 0
## Ported from AK/Utf16StringData.h:80 AK::Detail::Utf16StringData::hash (the hash fields are `mutable`).
pub func utf16_string_data_hash(this: const Utf16StringData*) -> u32:
if not this.m_has_hash:
let mutable_this = (Utf16StringData*)this
let view = utf16_string_data_utf16_view(this)
mutable_this.m_hash = utf16_view_hash(&view)
mutable_this.m_has_hash = true
return this.m_hash
## Ported from AK/Utf16StringData.h:90 AK::Detail::Utf16StringData::length_in_code_units.
pub func utf16_string_data_length_in_code_units(this: const Utf16StringData*) -> usize:
return this.m_length_in_code_units & ~((usize)1 << utf16_flag)
## Ported from AK/Utf16StringData.h:91 AK::Detail::Utf16StringData::length_in_code_points (the
## count is `mutable` and cached here).
pub func utf16_string_data_length_in_code_points(this: const Utf16StringData*) -> usize:
if utf16_string_data_has_ascii_storage(this):
return utf16_string_data_length_in_code_units(this)
if this.m_length_in_code_points == usize_max_code_points:
let mutable_this = (Utf16StringData*)this
mutable_this.m_length_in_code_points = utf16_string_data_calculate_code_point_length(this)
return this.m_length_in_code_points
## Ported from AK/Utf16StringData.h:100 AK::Detail::Utf16StringData::ascii_view.
pub func utf16_string_data_ascii_view(this: const Utf16StringData*) -> StringView:
assert(utf16_string_data_has_ascii_storage(this))
return string_view_from_bytes(u8[]((u8*)this + utf16_string_data_offset_of_string_storage(), utf16_string_data_length_in_code_units(this)))
## Ported from AK/Utf16StringData.h:106 AK::Detail::Utf16StringData::utf16_view.
pub func utf16_string_data_utf16_view(this: const Utf16StringData*) -> Utf16View:
let storage = (u8*)this + utf16_string_data_offset_of_string_storage()
if utf16_string_data_has_ascii_storage(this):
return utf16_view_from_ascii(u8[](storage, utf16_string_data_length_in_code_units(this)))
var view = utf16_view_from_utf16(u16[]((u16*)storage, utf16_string_data_length_in_code_units(this)))
view.m_length_in_code_points = this.m_length_in_code_points
return view
## Ported from AK/Utf16StringData.h:117 AK::Detail::Utf16StringData::mark_as_fly_string (a
## `mutable` field).
pub func utf16_string_data_mark_as_fly_string(this: const Utf16StringData*):
let mutable_this = (Utf16StringData*)this
mutable_this.m_is_fly_string = true
## Ported from AK/Utf16StringData.h:118 AK::Detail::Utf16StringData::is_fly_string.
pub func utf16_string_data_is_fly_string(this: const Utf16StringData*) -> bool:
return this.m_is_fly_string
## The ASCII storage of a string being filled.
func utf16_string_data_ascii_storage(this: Utf16StringData*) -> u8[]:
return u8[]((u8*)this + utf16_string_data_offset_of_string_storage(), utf16_string_data_length_in_code_units(this))
## The UTF-16 storage of a string being filled.
func utf16_string_data_utf16_storage(this: Utf16StringData*) -> u16[]:
return u16[]((u16*)((u8*)this + utf16_string_data_offset_of_string_storage()), utf16_string_data_length_in_code_units(this))
# mark: Construction ============================================================================
## Ported from AK/Utf16StringData.h:121 AK::Detail::Utf16StringData::Utf16StringData.
func utf16_string_data_construct(this: Utf16StringData*, storage_type: Utf16StringDataStorageType, code_unit_length: usize):
this.m_length_in_code_units = code_unit_length
this.m_length_in_code_points = usize_max_code_points
this.m_hash = 0
this.m_has_hash = false
this.m_is_fly_string = false
if storage_type == Utf16StringDataStorageType.utf16:
this.m_length_in_code_units |= (usize)1 << utf16_flag
## Ported from AK/Utf16StringData.cpp:18 AK::Detail::Utf16StringData::create_uninitialized.
func utf16_string_data_create_uninitialized(storage_type: Utf16StringDataStorageType, code_unit_length: usize) -> Utf16StringData*:
let allocation_size = utf16_string_data_allocation_size_for_string_data(storage_type == Utf16StringDataStorageType.ascii, code_unit_length)
let slot = memory.allocate(allocation_size, memory.align_of(Utf16StringData)) else trap("VERIFY: kmalloc")
let string = (Utf16StringData*)slot.data
utf16_string_data_construct(string, storage_type, code_unit_length)
return string
## Ported from AK/Utf16StringData.cpp:28 AK::Detail::Utf16StringData::create_from_code_point_iterable
## over a Utf8View: UTF-16 storage for its code points (invalid bytes become U+FFFD).
func utf16_string_data_create_from_code_point_iterable_utf8_view(view: Utf8View) -> Utf16StringData*:
var code_unit_length: usize = 0
var code_point_length: usize = 0
var it = utf8_view_begin(&view)
while not utf8_code_point_iterator_done(&it):
code_unit_length += unicode_utils_code_unit_length_for_code_point(utf8_code_point_iterator_deref(&it))
code_point_length += 1
_ = utf8_code_point_iterator_inc(&it)
verify_utf16_length(code_unit_length)
let string = utf16_string_data_create_uninitialized(Utf16StringDataStorageType.utf16, code_unit_length)
string.m_length_in_code_points = code_point_length
var writer = Utf16StringDataCodeUnitWriter(storage = utf16_string_data_utf16_storage(string), code_unit_index = 0)
it = utf8_view_begin(&view)
while not utf8_code_point_iterator_done(&it):
_ = unicode_utils_code_point_to_utf16(utf8_code_point_iterator_deref(&it), &writer)
_ = utf8_code_point_iterator_inc(&it)
return string
## Ported from AK/Utf16StringData.cpp:28 AK::Detail::Utf16StringData::create_from_code_point_iterable
## over a Utf32View.
func utf16_string_data_create_from_code_point_iterable_utf32_view(view: Utf32View) -> Utf16StringData*:
var code_unit_length: usize = 0
var code_point_length: usize = 0
for code_point in utf32_view_span(&view):
code_unit_length += unicode_utils_code_unit_length_for_code_point(code_point)
code_point_length += 1
verify_utf16_length(code_unit_length)
let string = utf16_string_data_create_uninitialized(Utf16StringDataStorageType.utf16, code_unit_length)
string.m_length_in_code_points = code_point_length
var writer = Utf16StringDataCodeUnitWriter(storage = utf16_string_data_utf16_storage(string), code_unit_index = 0)
for code_point in utf32_view_span(&view):
_ = unicode_utils_code_point_to_utf16(code_point, &writer)
return string
## The lambda of create_from_code_point_iterable: stores each code unit.
struct Utf16StringDataCodeUnitWriter: Utf16CodeUnitCallback:
var storage: u16[]
var code_unit_index: usize
func call(code_unit: u16):
self.storage[self.code_unit_index] = code_unit
self.code_unit_index += 1
## Ported from AK/Utf16StringData.h:134 AK::Detail::Utf16StringData::allocation_size_for_string_data.
func utf16_string_data_allocation_size_for_string_data(has_ascii_storage: bool, code_unit_length: usize) -> usize:
if has_ascii_storage:
return memory.size_of(Utf16StringData) + (memory.size_of(u8) * code_unit_length)
return memory.size_of(Utf16StringData) + (memory.size_of(u16) * code_unit_length)
## Ported from AK/Utf16StringData.cpp:203 AK::Detail::Utf16StringData::calculate_code_point_length.
func utf16_string_data_calculate_code_point_length(this: const Utf16StringData*) -> usize:
assert(not utf16_string_data_has_ascii_storage(this))
let view = utf16_string_data_utf16_view(this)
let code_units = utf16_view_utf16_span(&view)
if simdutf_validate_utf16(code_units):
return simdutf_count_utf16(code_units)
var code_points: usize = 0
var it = utf16_view_begin(&view)
while not utf16_code_point_iterator_done(&it):
code_points += 1
_ = utf16_code_point_iterator_inc(&it)
return code_points