Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.1

0.6.1 / src / web_unicode / segmenter.lucb

#==============================================================================================
#
#   segmenter - Unicode::Segmenter: grapheme, word, line and sentence segmentation
#
#   DESCRIPTION:
#       Port of LibUnicode/Segmenter.cpp (lines 1-356) and Segmenter.h (lines 1-66). ICU's
#       BreakIterator is web_unicode's BreakIterator (break_iterator.lucb); where the C++
#       hands ICU a UText or UnicodeString, the port hands it bytes or code units.
#
#==============================================================================================

# mark: Granularity names =======================================================================

## Ported from Libraries/LibUnicode/Segmenter.cpp:20 Unicode::segmenter_granularity_from_string.
pub func segmenter_granularity_from_string(segmenter_granularity: ak.StringView) -> SegmenterGranularity:
    if view_equals(segmenter_granularity, "grapheme"):
        return SegmenterGranularity.grapheme
    if view_equals(segmenter_granularity, "line"):
        return SegmenterGranularity.line
    if view_equals(segmenter_granularity, "sentence"):
        return SegmenterGranularity.sentence
    if view_equals(segmenter_granularity, "word"):
        return SegmenterGranularity.word
    trap("VERIFY_NOT_REACHED")

## Ported from Libraries/LibUnicode/Segmenter.cpp:33 Unicode::segmenter_granularity_to_string.
pub func segmenter_granularity_to_string(segmenter_granularity: SegmenterGranularity) -> ak.StringView:
    match segmenter_granularity:
        .grapheme: return view_of("grapheme")
        .line: return view_of("line")
        .sentence: return view_of("sentence")
        .word: return view_of("word")
        _: trap("VERIFY_NOT_REACHED")

# mark: AsciiGraphemeSegmenter ==================================================================

# Fast path segmenter for ASCII text where every character is its own grapheme.
# This avoids all ICU overhead for the common case of ASCII-only text.

## Ported from Libraries/LibUnicode/Segmenter.cpp:52 Unicode::AsciiGraphemeSegmenter::AsciiGraphemeSegmenter.
pub func ascii_grapheme_segmenter_construct(this: AsciiGraphemeSegmenter*, length: usize):
    segmenter_construct(this.segmenter(), SegmenterGranularity.grapheme)
    this.segmenter_base.vtable = (const SegmenterVTable*)&ascii_grapheme_segmenter_vtable
    this.segmenter_base.class_id = ascii_grapheme_segmenter_class.first_id
    this.m_length = length

## make<AsciiGraphemeSegmenter>(length).
func ascii_grapheme_segmenter_create(length: usize) -> Segmenter*:
    let this = new AsciiGraphemeSegmenter(segmenter_base = Segmenter(vtable = &segmenter_vtable, class_id = 0, m_segmenter_granularity = SegmenterGranularity.grapheme), m_length = 0, m_current = 0) else trap("out of memory")
    ascii_grapheme_segmenter_construct(this, length)
    return this.segmenter()

## Ported from Libraries/LibUnicode/Segmenter.cpp:60 Unicode::AsciiGraphemeSegmenter::clone (override).
pub func ascii_grapheme_segmenter_clone_impl(base: Segmenter*) -> Segmenter*:
    let this = (AsciiGraphemeSegmenter*)base
    return ascii_grapheme_segmenter_create(this.m_length)

## Ported from Libraries/LibUnicode/Segmenter.cpp:65 Unicode::AsciiGraphemeSegmenter::set_segmented_text (override).
pub func ascii_grapheme_segmenter_set_segmented_text_string_impl(base: Segmenter*, text: ak.String):
    let this = (AsciiGraphemeSegmenter*)base
    this.m_length = ak.string_base_byte_count(&text.string_base)

## Ported from Libraries/LibUnicode/Segmenter.cpp:70 Unicode::AsciiGraphemeSegmenter::set_segmented_text (override).
pub func ascii_grapheme_segmenter_set_segmented_text_utf16_view_impl(base: Segmenter*, text: ak.Utf16View):
    let this = (AsciiGraphemeSegmenter*)base
    this.m_length = utf16_view_length(&text)

## Ported from Libraries/LibUnicode/Segmenter.cpp:75 Unicode::AsciiGraphemeSegmenter::current_boundary (override).
pub func ascii_grapheme_segmenter_current_boundary_impl(base: Segmenter*) -> usize:
    let this = (AsciiGraphemeSegmenter*)base
    return this.m_current

## Ported from Libraries/LibUnicode/Segmenter.cpp:80 Unicode::AsciiGraphemeSegmenter::previous_boundary (override).
pub func ascii_grapheme_segmenter_previous_boundary_impl(base: Segmenter*, index: usize, inclusive: SegmenterInclusive) -> u64?:
    let this = (AsciiGraphemeSegmenter*)base
    if inclusive == SegmenterInclusive.yes and index <= this.m_length:
        return (u64)index
    if index == 0:
        return none
    return (u64)(index - 1)

## Ported from Libraries/LibUnicode/Segmenter.cpp:89 Unicode::AsciiGraphemeSegmenter::next_boundary (override).
pub func ascii_grapheme_segmenter_next_boundary_impl(base: Segmenter*, index: usize, inclusive: SegmenterInclusive) -> u64?:
    let this = (AsciiGraphemeSegmenter*)base
    if inclusive == SegmenterInclusive.yes and index <= this.m_length:
        return (u64)index
    if index >= this.m_length:
        return none
    return (u64)(index + 1)

## Ported from Libraries/LibUnicode/Segmenter.cpp:98 Unicode::AsciiGraphemeSegmenter::for_each_boundary (override).
pub func ascii_grapheme_segmenter_for_each_boundary_string_impl(base: Segmenter*, text: ak.String, callback: ak.Function1[u64, ak.IterationDecision]):
    let this = (AsciiGraphemeSegmenter*)base
    ascii_grapheme_segmenter_set_segmented_text_string_impl(base, text)
    var callback_ = callback
    ascii_grapheme_segmenter_for_each_boundary_impl(this, &callback_)

## Ported from Libraries/LibUnicode/Segmenter.cpp:104 Unicode::AsciiGraphemeSegmenter::for_each_boundary (override).
pub func ascii_grapheme_segmenter_for_each_boundary_utf16_view_impl(base: Segmenter*, text: ak.Utf16View, callback: ak.Function1[u64, ak.IterationDecision]):
    let this = (AsciiGraphemeSegmenter*)base
    ascii_grapheme_segmenter_set_segmented_text_utf16_view_impl(base, text)
    var callback_ = callback
    ascii_grapheme_segmenter_for_each_boundary_impl(this, &callback_)

## Ported from Libraries/LibUnicode/Segmenter.cpp:110 Unicode::AsciiGraphemeSegmenter::for_each_boundary (override).
pub func ascii_grapheme_segmenter_for_each_boundary_utf32_view_impl(base: Segmenter*, text: ak.Utf32View, callback: ak.Function1[u64, ak.IterationDecision]):
    let this = (AsciiGraphemeSegmenter*)base
    this.m_length = text.m_length
    var callback_ = callback
    ascii_grapheme_segmenter_for_each_boundary_impl(this, &callback_)

## Ported from Libraries/LibUnicode/Segmenter.cpp:116 Unicode::AsciiGraphemeSegmenter::is_current_boundary_word_like (override).
pub func ascii_grapheme_segmenter_is_current_boundary_word_like_impl(base: Segmenter*) -> bool:
    return false

## Ported from Libraries/LibUnicode/Segmenter.cpp:122 Unicode::AsciiGraphemeSegmenter::for_each_boundary_impl.
pub func ascii_grapheme_segmenter_for_each_boundary_impl(this: AsciiGraphemeSegmenter*, callback: ak.Function1[u64, ak.IterationDecision]*):
    for i in 0..=this.m_length:
        if call_boundary_callback(callback, i) == ak.IterationDecision.break_:
            return

# mark: SegmenterImpl ===========================================================================

## Ported from Libraries/LibUnicode/Segmenter.cpp:136 Unicode::SegmenterImpl::SegmenterImpl.
pub func segmenter_impl_construct(this: SegmenterImpl*, segmenter: BreakIterator*, segmenter_granularity: SegmenterGranularity):
    segmenter_construct(this.segmenter(), segmenter_granularity)
    this.segmenter_base.vtable = (const SegmenterVTable*)&segmenter_impl_vtable
    this.segmenter_base.class_id = segmenter_impl_class.first_id
    this.m_segmenter = segmenter

## make<SegmenterImpl>(segmenter, segmenter_granularity).
func segmenter_impl_create(segmenter: BreakIterator*, segmenter_granularity: SegmenterGranularity) -> Segmenter*:
    let this = new SegmenterImpl(segmenter_base = Segmenter(vtable = &segmenter_vtable, class_id = 0, m_segmenter_granularity = segmenter_granularity), m_segmenter = segmenter, m_segmented_text = SegmentedText.empty) else trap("out of memory")
    segmenter_impl_construct(this, segmenter, segmenter_granularity)
    return this.segmenter()

## Ported from Libraries/LibUnicode/Segmenter.cpp:144 Unicode::SegmenterImpl::clone (override).
pub func segmenter_impl_clone_impl(base: Segmenter*) -> Segmenter*:
    let this = (SegmenterImpl*)base
    let clone = segmenter_impl_create(break_iterator_clone(this.m_segmenter), this.segmenter_base.m_segmenter_granularity)
    # icu::BreakIterator::clone keeps the text; the port's iterator keeps only boundaries, so
    # the clone shares the text too.
    ((SegmenterImpl*)clone).m_segmented_text = this.m_segmented_text
    return clone

## Ported from Libraries/LibUnicode/Segmenter.cpp:149 Unicode::SegmenterImpl::set_segmented_text (override).
pub func segmenter_impl_set_segmented_text_string_impl(base: Segmenter*, text: ak.String):
    let this = (SegmenterImpl*)base
    let bytes = ak.string_base_bytes(&text.string_base)
    this.m_segmented_text = SegmentedText.utf8(string = text, bytes = bytes)
    # utext_openUTF8 + setText
    break_iterator_set_text_utf8(this.m_segmenter, bytes)

## Ported from Libraries/LibUnicode/Segmenter.cpp:166 Unicode::SegmenterImpl::set_segmented_text (override).
pub func segmenter_impl_set_segmented_text_utf16_view_impl(base: Segmenter*, text: ak.Utf16View):
    let this = (SegmenterImpl*)base
    if utf16_view_is_ascii(&text):
        # C++: set_segmented_text(MUST(text.to_utf8())). ASCII is its own UTF-8, so the port
        # segments a copy of the bytes, which gives the same offsets.
        let ascii = utf16_view_ascii(&text)
        let copy = new u8[ascii.length] else trap("out of memory")
        for index in 0..<ascii.length:
            copy[index] = ascii[index]
        this.m_segmented_text = SegmentedText.utf8(string = none, bytes = copy)
        break_iterator_set_text_utf8(this.m_segmenter, copy)
        return
    # icu::UnicodeString { text.utf16_span().data(), length }: a copy of the code units.
    let units = utf16_view_units(&text)
    let copy = new u16[units.length] else trap("out of memory")
    for index in 0..<units.length:
        copy[index] = units[index]
    this.m_segmented_text = SegmentedText.utf16(units = copy)
    break_iterator_set_text_utf16(this.m_segmenter, copy)

## Ported from Libraries/LibUnicode/Segmenter.cpp:177 Unicode::SegmenterImpl::current_boundary (override).
pub func segmenter_impl_current_boundary_impl(base: Segmenter*) -> usize:
    let this = (SegmenterImpl*)base
    return (usize)break_iterator_current(this.m_segmenter)

## Ported from Libraries/LibUnicode/Segmenter.cpp:182 Unicode::SegmenterImpl::previous_boundary (override).
pub func segmenter_impl_previous_boundary_impl(base: Segmenter*, boundary: usize, inclusive: SegmenterInclusive) -> u64?:
    let this = (SegmenterImpl*)base
    let icu_boundary = segmenter_impl_align_boundary(this, boundary)

    if inclusive == SegmenterInclusive.yes:
        if break_iterator_is_boundary(this.m_segmenter, icu_boundary):
            return (u64)icu_boundary

    let index = break_iterator_preceding(this.m_segmenter, icu_boundary)
    if index != break_iterator_done:
        return (u64)index

    return none

## Ported from Libraries/LibUnicode/Segmenter.cpp:197 Unicode::SegmenterImpl::next_boundary (override).
pub func segmenter_impl_next_boundary_impl(base: Segmenter*, boundary: usize, inclusive: SegmenterInclusive) -> u64?:
    let this = (SegmenterImpl*)base
    let icu_boundary = segmenter_impl_align_boundary(this, boundary)

    if inclusive == SegmenterInclusive.yes:
        if break_iterator_is_boundary(this.m_segmenter, icu_boundary):
            return (u64)icu_boundary

    let index = break_iterator_following(this.m_segmenter, icu_boundary)
    if index != break_iterator_done:
        return (u64)index

    return none

## Ported from Libraries/LibUnicode/Segmenter.cpp:212 Unicode::SegmenterImpl::for_each_boundary (override).
pub func segmenter_impl_for_each_boundary_string_impl(base: Segmenter*, text: ak.String, callback: ak.Function1[u64, ak.IterationDecision]):
    let this = (SegmenterImpl*)base
    if ak.string_base_byte_count(&text.string_base) == 0:
        return

    segmenter_impl_set_segmented_text_string_impl(base, text)
    segmenter_impl_for_each_boundary_function1(this, callback)

## Ported from Libraries/LibUnicode/Segmenter.cpp:221 Unicode::SegmenterImpl::for_each_boundary (override).
pub func segmenter_impl_for_each_boundary_utf16_view_impl(base: Segmenter*, text: ak.Utf16View, callback: ak.Function1[u64, ak.IterationDecision]):
    let this = (SegmenterImpl*)base
    if utf16_view_length(&text) == 0:
        return

    segmenter_impl_set_segmented_text_utf16_view_impl(base, text)
    segmenter_impl_for_each_boundary_function1(this, callback)

## The capture of SegmenterImpl::for_each_boundary(Utf32View)'s lambda: the byte offset of
## each code point, and the caller's callback.
struct SegmenterImplForEachBoundaryUtf32:
    var starts: const u32[]
    var code_point_index: usize
    var callback: ak.Function1[u64, ak.IterationDecision]

func segmenter_impl_for_each_boundary_utf32_run(context: void*?, index: u64) -> ak.IterationDecision:
    let captures = (SegmenterImplForEachBoundaryUtf32*)(context else trap("no capture"))
    # auto it = code_points.iterator_at_byte_offset(index);
    # while (current != it) { ++code_point_index; ++current; }
    while captures.code_point_index + 1 < captures.starts.length and (u64)captures.starts[captures.code_point_index] < index:
        captures.code_point_index += 1
    return call_boundary_callback(&captures.callback, captures.code_point_index)

## Ported from Libraries/LibUnicode/Segmenter.cpp:230 Unicode::SegmenterImpl::for_each_boundary (override).
pub func segmenter_impl_for_each_boundary_utf32_view_impl(base: Segmenter*, text: ak.Utf32View, callback: ak.Function1[u64, ak.IterationDecision]):
    let this = (SegmenterImpl*)base
    if text.m_length == 0:
        return

    # FIXME: We should be able to create a custom UText provider to avoid converting to UTF-8 here.
    # C++: set_segmented_text(MUST(String::formatted("{}", text))); the port encodes the code
    # points to UTF-8 itself, which is what formatting a Utf32View does.
    let code_points = utf32_view_code_points(&text)
    var bytes = ByteBuffer.create(code_points.length * 4 + 1)
    var starts = CodePointBuffer.create(code_points.length + 1)
    for code_point in code_points:
        starts.append((u32)bytes.count)
        encode_utf8(code_point, &bytes)
    starts.append((u32)bytes.count)
    this.m_segmented_text = SegmentedText.utf8(string = none, bytes = bytes.view())
    break_iterator_set_text_utf8(this.m_segmenter, bytes.view())

    var captures = SegmenterImplForEachBoundaryUtf32(starts = starts.view(), code_point_index = 0, callback = callback)
    let adapter = ak.Function1[u64, ak.IterationDecision](call = segmenter_impl_for_each_boundary_utf32_run, context = &captures)
    segmenter_impl_for_each_boundary_function1(this, adapter)

# UBRK_WORD_* rule status ranges (ubrk.h).
let ubrk_word_number_limit: i32 = 200
let ubrk_word_letter_limit: i32 = 300
let ubrk_word_kana_limit: i32 = 400
let ubrk_word_ideo_limit: i32 = 500

## Ported from Libraries/LibUnicode/Segmenter.cpp:254 Unicode::SegmenterImpl::is_current_boundary_word_like (override).
pub func segmenter_impl_is_current_boundary_word_like_impl(base: Segmenter*) -> bool:
    let this = (SegmenterImpl*)base
    let status = break_iterator_rule_status(this.m_segmenter)

    if status >= (i32)ubrk_word_number and status < ubrk_word_number_limit:
        return true
    if status >= (i32)ubrk_word_letter and status < ubrk_word_letter_limit:
        return true
    if status >= (i32)ubrk_word_kana and status < ubrk_word_kana_limit:
        return true
    if status >= (i32)ubrk_word_ideo and status < ubrk_word_ideo_limit:
        return true

    return false

## Ported from Libraries/LibUnicode/Segmenter.cpp:271 Unicode::SegmenterImpl::align_boundary.
pub func segmenter_impl_align_boundary(this: SegmenterImpl*, boundary: usize) -> i32:
    match this.m_segmented_text:
        .utf8(_, bytes):
            if boundary >= bytes.length:
                return (i32)bytes.length

            # U8_SET_CP_START
            var icu_boundary = boundary
            while icu_boundary > 0 and (bytes[icu_boundary] & 0xc0) == 0x80:
                icu_boundary -= 1
            return (i32)icu_boundary
        .utf16(units):
            if boundary >= units.length:
                return (i32)units.length

            # getChar32Start
            var icu_boundary = boundary
            if icu_boundary > 0 and units[icu_boundary] >= 0xdc00 and units[icu_boundary] <= 0xdfff and units[icu_boundary - 1] >= 0xd800 and units[icu_boundary - 1] <= 0xdbff:
                icu_boundary -= 1
            return (i32)icu_boundary
        .empty:
            trap("VERIFY_NOT_REACHED")

## Ported from Libraries/LibUnicode/Segmenter.cpp:292 Unicode::SegmenterImpl::for_each_boundary.
pub func segmenter_impl_for_each_boundary_function1(this: SegmenterImpl*, callback: ak.Function1[u64, ak.IterationDecision]):
    var callback_ = callback
    if call_boundary_callback(&callback_, (usize)break_iterator_first(this.m_segmenter)) == ak.IterationDecision.break_:
        return

    while true:
        let index = break_iterator_next(this.m_segmenter)
        if index == break_iterator_done:
            return

        if call_boundary_callback(&callback_, (usize)index) == ak.IterationDecision.break_:
            return

# mark: Segmenter ===============================================================================

## Ported from Libraries/LibUnicode/Segmenter.cpp:311 Unicode::Segmenter::create.
pub func segmenter_create_segmenter_granularity(segmenter_granularity: SegmenterGranularity) -> Segmenter*:
    return segmenter_create_string_view(default_locale(), segmenter_granularity)

## Ported from Libraries/LibUnicode/Segmenter.cpp:316 Unicode::Segmenter::create.
## The port's rules are the root locale's for every locale (ICU tailors a few locales).
pub func segmenter_create_string_view(locale: ak.StringView, segmenter_granularity: SegmenterGranularity) -> Segmenter*:
    _ = locale
    # createCharacterInstance, createLineInstance, createSentenceInstance, createWordInstance
    let segmenter = break_iterator_create(segmenter_granularity)
    return segmenter_impl_create(segmenter, segmenter_granularity)

## Ported from Libraries/LibUnicode/Segmenter.cpp:342 Unicode::Segmenter::create_for_ascii_grapheme.
pub func segmenter_create_for_ascii_grapheme(length: usize) -> Segmenter*:
    return ascii_grapheme_segmenter_create(length)

## Ported from Libraries/LibUnicode/Segmenter.cpp:347 Unicode::Segmenter::should_continue_beyond_word.
pub func segmenter_should_continue_beyond_word(word: ak.Utf16View) -> bool:
    var code_points = CodePointBuffer.create(utf16_view_length(&word) + 1)
    if utf16_view_is_ascii(&word):
        for byte in utf16_view_ascii(&word):
            code_points.append((u32)byte)
    else:
        decode_utf16(utf16_view_units(&word), &code_points, none)
    for code_point in code_points.view():
        if not code_point_has_punctuation_general_category(code_point) and not code_point_has_separator_general_category(code_point):
            return false
    return true

## Ported from Libraries/LibUnicode/Segmenter.h:34 Unicode::Segmenter::segmenter_granularity.
pub func segmenter_segmenter_granularity(this: Segmenter*) -> SegmenterGranularity:
    return this.m_segmenter_granularity

## Ported from Libraries/LibUnicode/Segmenter.h:36 Unicode::Segmenter::clone (virtual).
pub func segmenter_clone_impl(this: Segmenter*) -> Segmenter*:
    trap("pure virtual: Unicode::Segmenter::clone")

## Ported from Libraries/LibUnicode/Segmenter.h:38 Unicode::Segmenter::set_segmented_text (virtual).
pub func segmenter_set_segmented_text_string_impl(this: Segmenter*, arg0: ak.String):
    trap("pure virtual: Unicode::Segmenter::set_segmented_text")

## Ported from Libraries/LibUnicode/Segmenter.h:39 Unicode::Segmenter::set_segmented_text (virtual).
pub func segmenter_set_segmented_text_utf16_view_impl(this: Segmenter*, arg0: ak.Utf16View):
    trap("pure virtual: Unicode::Segmenter::set_segmented_text")

## Ported from Libraries/LibUnicode/Segmenter.h:41 Unicode::Segmenter::current_boundary (virtual).
pub func segmenter_current_boundary_impl(this: Segmenter*) -> usize:
    trap("pure virtual: Unicode::Segmenter::current_boundary")

## Ported from Libraries/LibUnicode/Segmenter.h:47 Unicode::Segmenter::previous_boundary (virtual).
pub func segmenter_previous_boundary_impl(this: Segmenter*, index: usize, arg1: SegmenterInclusive) -> u64?:
    trap("pure virtual: Unicode::Segmenter::previous_boundary")

## Ported from Libraries/LibUnicode/Segmenter.h:48 Unicode::Segmenter::next_boundary (virtual).
pub func segmenter_next_boundary_impl(this: Segmenter*, index: usize, arg1: SegmenterInclusive) -> u64?:
    trap("pure virtual: Unicode::Segmenter::next_boundary")

## Ported from Libraries/LibUnicode/Segmenter.h:51 Unicode::Segmenter::for_each_boundary (virtual).
pub func segmenter_for_each_boundary_string_impl(this: Segmenter*, arg0: ak.String, arg1: ak.Function1[u64, ak.IterationDecision]):
    trap("pure virtual: Unicode::Segmenter::for_each_boundary")

## Ported from Libraries/LibUnicode/Segmenter.h:52 Unicode::Segmenter::for_each_boundary (virtual).
pub func segmenter_for_each_boundary_utf16_view_impl(this: Segmenter*, arg0: ak.Utf16View, arg1: ak.Function1[u64, ak.IterationDecision]):
    trap("pure virtual: Unicode::Segmenter::for_each_boundary")

## Ported from Libraries/LibUnicode/Segmenter.h:53 Unicode::Segmenter::for_each_boundary (virtual).
pub func segmenter_for_each_boundary_utf32_view_impl(this: Segmenter*, arg0: ak.Utf32View, arg1: ak.Function1[u64, ak.IterationDecision]):
    trap("pure virtual: Unicode::Segmenter::for_each_boundary")

## Ported from Libraries/LibUnicode/Segmenter.h:55 Unicode::Segmenter::is_current_boundary_word_like (virtual).
pub func segmenter_is_current_boundary_word_like_impl(this: Segmenter*) -> bool:
    trap("pure virtual: Unicode::Segmenter::is_current_boundary_word_like")

## Ported from Libraries/LibUnicode/Segmenter.h:58 Unicode::Segmenter::Segmenter.
pub func segmenter_construct(this: Segmenter*, segmenter_granularity: SegmenterGranularity):
    this.m_segmenter_granularity = segmenter_granularity