dymokomi/luce-browser-foundation
Source of release 0.6.1
0.6.1 / src / web_unicode / segmenter.lucb
#==============================================================================================
#
# segmenter - Unicode::Segmenter: grapheme, word, line and sentence segmentation
#
# DESCRIPTION:
# Port of LibUnicode/Segmenter.cpp (lines 1-356) and Segmenter.h (lines 1-66). ICU's
# BreakIterator is web_unicode's BreakIterator (break_iterator.lucb); where the C++
# hands ICU a UText or UnicodeString, the port hands it bytes or code units.
#
#==============================================================================================
# mark: Granularity names =======================================================================
## Ported from Libraries/LibUnicode/Segmenter.cpp:20 Unicode::segmenter_granularity_from_string.
pub func segmenter_granularity_from_string(segmenter_granularity: ak.StringView) -> SegmenterGranularity:
if view_equals(segmenter_granularity, "grapheme"):
return SegmenterGranularity.grapheme
if view_equals(segmenter_granularity, "line"):
return SegmenterGranularity.line
if view_equals(segmenter_granularity, "sentence"):
return SegmenterGranularity.sentence
if view_equals(segmenter_granularity, "word"):
return SegmenterGranularity.word
trap("VERIFY_NOT_REACHED")
## Ported from Libraries/LibUnicode/Segmenter.cpp:33 Unicode::segmenter_granularity_to_string.
pub func segmenter_granularity_to_string(segmenter_granularity: SegmenterGranularity) -> ak.StringView:
match segmenter_granularity:
.grapheme: return view_of("grapheme")
.line: return view_of("line")
.sentence: return view_of("sentence")
.word: return view_of("word")
_: trap("VERIFY_NOT_REACHED")
# mark: AsciiGraphemeSegmenter ==================================================================
# Fast path segmenter for ASCII text where every character is its own grapheme.
# This avoids all ICU overhead for the common case of ASCII-only text.
## Ported from Libraries/LibUnicode/Segmenter.cpp:52 Unicode::AsciiGraphemeSegmenter::AsciiGraphemeSegmenter.
pub func ascii_grapheme_segmenter_construct(this: AsciiGraphemeSegmenter*, length: usize):
segmenter_construct(this.segmenter(), SegmenterGranularity.grapheme)
this.segmenter_base.vtable = (const SegmenterVTable*)&ascii_grapheme_segmenter_vtable
this.segmenter_base.class_id = ascii_grapheme_segmenter_class.first_id
this.m_length = length
## make<AsciiGraphemeSegmenter>(length).
func ascii_grapheme_segmenter_create(length: usize) -> Segmenter*:
let this = new AsciiGraphemeSegmenter(segmenter_base = Segmenter(vtable = &segmenter_vtable, class_id = 0, m_segmenter_granularity = SegmenterGranularity.grapheme), m_length = 0, m_current = 0) else trap("out of memory")
ascii_grapheme_segmenter_construct(this, length)
return this.segmenter()
## Ported from Libraries/LibUnicode/Segmenter.cpp:60 Unicode::AsciiGraphemeSegmenter::clone (override).
pub func ascii_grapheme_segmenter_clone_impl(base: Segmenter*) -> Segmenter*:
let this = (AsciiGraphemeSegmenter*)base
return ascii_grapheme_segmenter_create(this.m_length)
## Ported from Libraries/LibUnicode/Segmenter.cpp:65 Unicode::AsciiGraphemeSegmenter::set_segmented_text (override).
pub func ascii_grapheme_segmenter_set_segmented_text_string_impl(base: Segmenter*, text: ak.String):
let this = (AsciiGraphemeSegmenter*)base
this.m_length = ak.string_base_byte_count(&text.string_base)
## Ported from Libraries/LibUnicode/Segmenter.cpp:70 Unicode::AsciiGraphemeSegmenter::set_segmented_text (override).
pub func ascii_grapheme_segmenter_set_segmented_text_utf16_view_impl(base: Segmenter*, text: ak.Utf16View):
let this = (AsciiGraphemeSegmenter*)base
this.m_length = utf16_view_length(&text)
## Ported from Libraries/LibUnicode/Segmenter.cpp:75 Unicode::AsciiGraphemeSegmenter::current_boundary (override).
pub func ascii_grapheme_segmenter_current_boundary_impl(base: Segmenter*) -> usize:
let this = (AsciiGraphemeSegmenter*)base
return this.m_current
## Ported from Libraries/LibUnicode/Segmenter.cpp:80 Unicode::AsciiGraphemeSegmenter::previous_boundary (override).
pub func ascii_grapheme_segmenter_previous_boundary_impl(base: Segmenter*, index: usize, inclusive: SegmenterInclusive) -> u64?:
let this = (AsciiGraphemeSegmenter*)base
if inclusive == SegmenterInclusive.yes and index <= this.m_length:
return (u64)index
if index == 0:
return none
return (u64)(index - 1)
## Ported from Libraries/LibUnicode/Segmenter.cpp:89 Unicode::AsciiGraphemeSegmenter::next_boundary (override).
pub func ascii_grapheme_segmenter_next_boundary_impl(base: Segmenter*, index: usize, inclusive: SegmenterInclusive) -> u64?:
let this = (AsciiGraphemeSegmenter*)base
if inclusive == SegmenterInclusive.yes and index <= this.m_length:
return (u64)index
if index >= this.m_length:
return none
return (u64)(index + 1)
## Ported from Libraries/LibUnicode/Segmenter.cpp:98 Unicode::AsciiGraphemeSegmenter::for_each_boundary (override).
pub func ascii_grapheme_segmenter_for_each_boundary_string_impl(base: Segmenter*, text: ak.String, callback: ak.Function1[u64, ak.IterationDecision]):
let this = (AsciiGraphemeSegmenter*)base
ascii_grapheme_segmenter_set_segmented_text_string_impl(base, text)
var callback_ = callback
ascii_grapheme_segmenter_for_each_boundary_impl(this, &callback_)
## Ported from Libraries/LibUnicode/Segmenter.cpp:104 Unicode::AsciiGraphemeSegmenter::for_each_boundary (override).
pub func ascii_grapheme_segmenter_for_each_boundary_utf16_view_impl(base: Segmenter*, text: ak.Utf16View, callback: ak.Function1[u64, ak.IterationDecision]):
let this = (AsciiGraphemeSegmenter*)base
ascii_grapheme_segmenter_set_segmented_text_utf16_view_impl(base, text)
var callback_ = callback
ascii_grapheme_segmenter_for_each_boundary_impl(this, &callback_)
## Ported from Libraries/LibUnicode/Segmenter.cpp:110 Unicode::AsciiGraphemeSegmenter::for_each_boundary (override).
pub func ascii_grapheme_segmenter_for_each_boundary_utf32_view_impl(base: Segmenter*, text: ak.Utf32View, callback: ak.Function1[u64, ak.IterationDecision]):
let this = (AsciiGraphemeSegmenter*)base
this.m_length = text.m_length
var callback_ = callback
ascii_grapheme_segmenter_for_each_boundary_impl(this, &callback_)
## Ported from Libraries/LibUnicode/Segmenter.cpp:116 Unicode::AsciiGraphemeSegmenter::is_current_boundary_word_like (override).
pub func ascii_grapheme_segmenter_is_current_boundary_word_like_impl(base: Segmenter*) -> bool:
return false
## Ported from Libraries/LibUnicode/Segmenter.cpp:122 Unicode::AsciiGraphemeSegmenter::for_each_boundary_impl.
pub func ascii_grapheme_segmenter_for_each_boundary_impl(this: AsciiGraphemeSegmenter*, callback: ak.Function1[u64, ak.IterationDecision]*):
for i in 0..=this.m_length:
if call_boundary_callback(callback, i) == ak.IterationDecision.break_:
return
# mark: SegmenterImpl ===========================================================================
## Ported from Libraries/LibUnicode/Segmenter.cpp:136 Unicode::SegmenterImpl::SegmenterImpl.
pub func segmenter_impl_construct(this: SegmenterImpl*, segmenter: BreakIterator*, segmenter_granularity: SegmenterGranularity):
segmenter_construct(this.segmenter(), segmenter_granularity)
this.segmenter_base.vtable = (const SegmenterVTable*)&segmenter_impl_vtable
this.segmenter_base.class_id = segmenter_impl_class.first_id
this.m_segmenter = segmenter
## make<SegmenterImpl>(segmenter, segmenter_granularity).
func segmenter_impl_create(segmenter: BreakIterator*, segmenter_granularity: SegmenterGranularity) -> Segmenter*:
let this = new SegmenterImpl(segmenter_base = Segmenter(vtable = &segmenter_vtable, class_id = 0, m_segmenter_granularity = segmenter_granularity), m_segmenter = segmenter, m_segmented_text = SegmentedText.empty) else trap("out of memory")
segmenter_impl_construct(this, segmenter, segmenter_granularity)
return this.segmenter()
## Ported from Libraries/LibUnicode/Segmenter.cpp:144 Unicode::SegmenterImpl::clone (override).
pub func segmenter_impl_clone_impl(base: Segmenter*) -> Segmenter*:
let this = (SegmenterImpl*)base
let clone = segmenter_impl_create(break_iterator_clone(this.m_segmenter), this.segmenter_base.m_segmenter_granularity)
# icu::BreakIterator::clone keeps the text; the port's iterator keeps only boundaries, so
# the clone shares the text too.
((SegmenterImpl*)clone).m_segmented_text = this.m_segmented_text
return clone
## Ported from Libraries/LibUnicode/Segmenter.cpp:149 Unicode::SegmenterImpl::set_segmented_text (override).
pub func segmenter_impl_set_segmented_text_string_impl(base: Segmenter*, text: ak.String):
let this = (SegmenterImpl*)base
let bytes = ak.string_base_bytes(&text.string_base)
this.m_segmented_text = SegmentedText.utf8(string = text, bytes = bytes)
# utext_openUTF8 + setText
break_iterator_set_text_utf8(this.m_segmenter, bytes)
## Ported from Libraries/LibUnicode/Segmenter.cpp:166 Unicode::SegmenterImpl::set_segmented_text (override).
pub func segmenter_impl_set_segmented_text_utf16_view_impl(base: Segmenter*, text: ak.Utf16View):
let this = (SegmenterImpl*)base
if utf16_view_is_ascii(&text):
# C++: set_segmented_text(MUST(text.to_utf8())). ASCII is its own UTF-8, so the port
# segments a copy of the bytes, which gives the same offsets.
let ascii = utf16_view_ascii(&text)
let copy = new u8[ascii.length] else trap("out of memory")
for index in 0..<ascii.length:
copy[index] = ascii[index]
this.m_segmented_text = SegmentedText.utf8(string = none, bytes = copy)
break_iterator_set_text_utf8(this.m_segmenter, copy)
return
# icu::UnicodeString { text.utf16_span().data(), length }: a copy of the code units.
let units = utf16_view_units(&text)
let copy = new u16[units.length] else trap("out of memory")
for index in 0..<units.length:
copy[index] = units[index]
this.m_segmented_text = SegmentedText.utf16(units = copy)
break_iterator_set_text_utf16(this.m_segmenter, copy)
## Ported from Libraries/LibUnicode/Segmenter.cpp:177 Unicode::SegmenterImpl::current_boundary (override).
pub func segmenter_impl_current_boundary_impl(base: Segmenter*) -> usize:
let this = (SegmenterImpl*)base
return (usize)break_iterator_current(this.m_segmenter)
## Ported from Libraries/LibUnicode/Segmenter.cpp:182 Unicode::SegmenterImpl::previous_boundary (override).
pub func segmenter_impl_previous_boundary_impl(base: Segmenter*, boundary: usize, inclusive: SegmenterInclusive) -> u64?:
let this = (SegmenterImpl*)base
let icu_boundary = segmenter_impl_align_boundary(this, boundary)
if inclusive == SegmenterInclusive.yes:
if break_iterator_is_boundary(this.m_segmenter, icu_boundary):
return (u64)icu_boundary
let index = break_iterator_preceding(this.m_segmenter, icu_boundary)
if index != break_iterator_done:
return (u64)index
return none
## Ported from Libraries/LibUnicode/Segmenter.cpp:197 Unicode::SegmenterImpl::next_boundary (override).
pub func segmenter_impl_next_boundary_impl(base: Segmenter*, boundary: usize, inclusive: SegmenterInclusive) -> u64?:
let this = (SegmenterImpl*)base
let icu_boundary = segmenter_impl_align_boundary(this, boundary)
if inclusive == SegmenterInclusive.yes:
if break_iterator_is_boundary(this.m_segmenter, icu_boundary):
return (u64)icu_boundary
let index = break_iterator_following(this.m_segmenter, icu_boundary)
if index != break_iterator_done:
return (u64)index
return none
## Ported from Libraries/LibUnicode/Segmenter.cpp:212 Unicode::SegmenterImpl::for_each_boundary (override).
pub func segmenter_impl_for_each_boundary_string_impl(base: Segmenter*, text: ak.String, callback: ak.Function1[u64, ak.IterationDecision]):
let this = (SegmenterImpl*)base
if ak.string_base_byte_count(&text.string_base) == 0:
return
segmenter_impl_set_segmented_text_string_impl(base, text)
segmenter_impl_for_each_boundary_function1(this, callback)
## Ported from Libraries/LibUnicode/Segmenter.cpp:221 Unicode::SegmenterImpl::for_each_boundary (override).
pub func segmenter_impl_for_each_boundary_utf16_view_impl(base: Segmenter*, text: ak.Utf16View, callback: ak.Function1[u64, ak.IterationDecision]):
let this = (SegmenterImpl*)base
if utf16_view_length(&text) == 0:
return
segmenter_impl_set_segmented_text_utf16_view_impl(base, text)
segmenter_impl_for_each_boundary_function1(this, callback)
## The capture of SegmenterImpl::for_each_boundary(Utf32View)'s lambda: the byte offset of
## each code point, and the caller's callback.
struct SegmenterImplForEachBoundaryUtf32:
var starts: const u32[]
var code_point_index: usize
var callback: ak.Function1[u64, ak.IterationDecision]
func segmenter_impl_for_each_boundary_utf32_run(context: void*?, index: u64) -> ak.IterationDecision:
let captures = (SegmenterImplForEachBoundaryUtf32*)(context else trap("no capture"))
# auto it = code_points.iterator_at_byte_offset(index);
# while (current != it) { ++code_point_index; ++current; }
while captures.code_point_index + 1 < captures.starts.length and (u64)captures.starts[captures.code_point_index] < index:
captures.code_point_index += 1
return call_boundary_callback(&captures.callback, captures.code_point_index)
## Ported from Libraries/LibUnicode/Segmenter.cpp:230 Unicode::SegmenterImpl::for_each_boundary (override).
pub func segmenter_impl_for_each_boundary_utf32_view_impl(base: Segmenter*, text: ak.Utf32View, callback: ak.Function1[u64, ak.IterationDecision]):
let this = (SegmenterImpl*)base
if text.m_length == 0:
return
# FIXME: We should be able to create a custom UText provider to avoid converting to UTF-8 here.
# C++: set_segmented_text(MUST(String::formatted("{}", text))); the port encodes the code
# points to UTF-8 itself, which is what formatting a Utf32View does.
let code_points = utf32_view_code_points(&text)
var bytes = ByteBuffer.create(code_points.length * 4 + 1)
var starts = CodePointBuffer.create(code_points.length + 1)
for code_point in code_points:
starts.append((u32)bytes.count)
encode_utf8(code_point, &bytes)
starts.append((u32)bytes.count)
this.m_segmented_text = SegmentedText.utf8(string = none, bytes = bytes.view())
break_iterator_set_text_utf8(this.m_segmenter, bytes.view())
var captures = SegmenterImplForEachBoundaryUtf32(starts = starts.view(), code_point_index = 0, callback = callback)
let adapter = ak.Function1[u64, ak.IterationDecision](call = segmenter_impl_for_each_boundary_utf32_run, context = &captures)
segmenter_impl_for_each_boundary_function1(this, adapter)
# UBRK_WORD_* rule status ranges (ubrk.h).
let ubrk_word_number_limit: i32 = 200
let ubrk_word_letter_limit: i32 = 300
let ubrk_word_kana_limit: i32 = 400
let ubrk_word_ideo_limit: i32 = 500
## Ported from Libraries/LibUnicode/Segmenter.cpp:254 Unicode::SegmenterImpl::is_current_boundary_word_like (override).
pub func segmenter_impl_is_current_boundary_word_like_impl(base: Segmenter*) -> bool:
let this = (SegmenterImpl*)base
let status = break_iterator_rule_status(this.m_segmenter)
if status >= (i32)ubrk_word_number and status < ubrk_word_number_limit:
return true
if status >= (i32)ubrk_word_letter and status < ubrk_word_letter_limit:
return true
if status >= (i32)ubrk_word_kana and status < ubrk_word_kana_limit:
return true
if status >= (i32)ubrk_word_ideo and status < ubrk_word_ideo_limit:
return true
return false
## Ported from Libraries/LibUnicode/Segmenter.cpp:271 Unicode::SegmenterImpl::align_boundary.
pub func segmenter_impl_align_boundary(this: SegmenterImpl*, boundary: usize) -> i32:
match this.m_segmented_text:
.utf8(_, bytes):
if boundary >= bytes.length:
return (i32)bytes.length
# U8_SET_CP_START
var icu_boundary = boundary
while icu_boundary > 0 and (bytes[icu_boundary] & 0xc0) == 0x80:
icu_boundary -= 1
return (i32)icu_boundary
.utf16(units):
if boundary >= units.length:
return (i32)units.length
# getChar32Start
var icu_boundary = boundary
if icu_boundary > 0 and units[icu_boundary] >= 0xdc00 and units[icu_boundary] <= 0xdfff and units[icu_boundary - 1] >= 0xd800 and units[icu_boundary - 1] <= 0xdbff:
icu_boundary -= 1
return (i32)icu_boundary
.empty:
trap("VERIFY_NOT_REACHED")
## Ported from Libraries/LibUnicode/Segmenter.cpp:292 Unicode::SegmenterImpl::for_each_boundary.
pub func segmenter_impl_for_each_boundary_function1(this: SegmenterImpl*, callback: ak.Function1[u64, ak.IterationDecision]):
var callback_ = callback
if call_boundary_callback(&callback_, (usize)break_iterator_first(this.m_segmenter)) == ak.IterationDecision.break_:
return
while true:
let index = break_iterator_next(this.m_segmenter)
if index == break_iterator_done:
return
if call_boundary_callback(&callback_, (usize)index) == ak.IterationDecision.break_:
return
# mark: Segmenter ===============================================================================
## Ported from Libraries/LibUnicode/Segmenter.cpp:311 Unicode::Segmenter::create.
pub func segmenter_create_segmenter_granularity(segmenter_granularity: SegmenterGranularity) -> Segmenter*:
return segmenter_create_string_view(default_locale(), segmenter_granularity)
## Ported from Libraries/LibUnicode/Segmenter.cpp:316 Unicode::Segmenter::create.
## The port's rules are the root locale's for every locale (ICU tailors a few locales).
pub func segmenter_create_string_view(locale: ak.StringView, segmenter_granularity: SegmenterGranularity) -> Segmenter*:
_ = locale
# createCharacterInstance, createLineInstance, createSentenceInstance, createWordInstance
let segmenter = break_iterator_create(segmenter_granularity)
return segmenter_impl_create(segmenter, segmenter_granularity)
## Ported from Libraries/LibUnicode/Segmenter.cpp:342 Unicode::Segmenter::create_for_ascii_grapheme.
pub func segmenter_create_for_ascii_grapheme(length: usize) -> Segmenter*:
return ascii_grapheme_segmenter_create(length)
## Ported from Libraries/LibUnicode/Segmenter.cpp:347 Unicode::Segmenter::should_continue_beyond_word.
pub func segmenter_should_continue_beyond_word(word: ak.Utf16View) -> bool:
var code_points = CodePointBuffer.create(utf16_view_length(&word) + 1)
if utf16_view_is_ascii(&word):
for byte in utf16_view_ascii(&word):
code_points.append((u32)byte)
else:
decode_utf16(utf16_view_units(&word), &code_points, none)
for code_point in code_points.view():
if not code_point_has_punctuation_general_category(code_point) and not code_point_has_separator_general_category(code_point):
return false
return true
## Ported from Libraries/LibUnicode/Segmenter.h:34 Unicode::Segmenter::segmenter_granularity.
pub func segmenter_segmenter_granularity(this: Segmenter*) -> SegmenterGranularity:
return this.m_segmenter_granularity
## Ported from Libraries/LibUnicode/Segmenter.h:36 Unicode::Segmenter::clone (virtual).
pub func segmenter_clone_impl(this: Segmenter*) -> Segmenter*:
trap("pure virtual: Unicode::Segmenter::clone")
## Ported from Libraries/LibUnicode/Segmenter.h:38 Unicode::Segmenter::set_segmented_text (virtual).
pub func segmenter_set_segmented_text_string_impl(this: Segmenter*, arg0: ak.String):
trap("pure virtual: Unicode::Segmenter::set_segmented_text")
## Ported from Libraries/LibUnicode/Segmenter.h:39 Unicode::Segmenter::set_segmented_text (virtual).
pub func segmenter_set_segmented_text_utf16_view_impl(this: Segmenter*, arg0: ak.Utf16View):
trap("pure virtual: Unicode::Segmenter::set_segmented_text")
## Ported from Libraries/LibUnicode/Segmenter.h:41 Unicode::Segmenter::current_boundary (virtual).
pub func segmenter_current_boundary_impl(this: Segmenter*) -> usize:
trap("pure virtual: Unicode::Segmenter::current_boundary")
## Ported from Libraries/LibUnicode/Segmenter.h:47 Unicode::Segmenter::previous_boundary (virtual).
pub func segmenter_previous_boundary_impl(this: Segmenter*, index: usize, arg1: SegmenterInclusive) -> u64?:
trap("pure virtual: Unicode::Segmenter::previous_boundary")
## Ported from Libraries/LibUnicode/Segmenter.h:48 Unicode::Segmenter::next_boundary (virtual).
pub func segmenter_next_boundary_impl(this: Segmenter*, index: usize, arg1: SegmenterInclusive) -> u64?:
trap("pure virtual: Unicode::Segmenter::next_boundary")
## Ported from Libraries/LibUnicode/Segmenter.h:51 Unicode::Segmenter::for_each_boundary (virtual).
pub func segmenter_for_each_boundary_string_impl(this: Segmenter*, arg0: ak.String, arg1: ak.Function1[u64, ak.IterationDecision]):
trap("pure virtual: Unicode::Segmenter::for_each_boundary")
## Ported from Libraries/LibUnicode/Segmenter.h:52 Unicode::Segmenter::for_each_boundary (virtual).
pub func segmenter_for_each_boundary_utf16_view_impl(this: Segmenter*, arg0: ak.Utf16View, arg1: ak.Function1[u64, ak.IterationDecision]):
trap("pure virtual: Unicode::Segmenter::for_each_boundary")
## Ported from Libraries/LibUnicode/Segmenter.h:53 Unicode::Segmenter::for_each_boundary (virtual).
pub func segmenter_for_each_boundary_utf32_view_impl(this: Segmenter*, arg0: ak.Utf32View, arg1: ak.Function1[u64, ak.IterationDecision]):
trap("pure virtual: Unicode::Segmenter::for_each_boundary")
## Ported from Libraries/LibUnicode/Segmenter.h:55 Unicode::Segmenter::is_current_boundary_word_like (virtual).
pub func segmenter_is_current_boundary_word_like_impl(this: Segmenter*) -> bool:
trap("pure virtual: Unicode::Segmenter::is_current_boundary_word_like")
## Ported from Libraries/LibUnicode/Segmenter.h:58 Unicode::Segmenter::Segmenter.
pub func segmenter_construct(this: Segmenter*, segmenter_granularity: SegmenterGranularity):
this.m_segmenter_granularity = segmenter_granularity