Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.0

0.6.0 / src / web_unicode / break_word.lucb

#==============================================================================================
#
#   break_word - word boundaries and word rule statuses (UAX #29, ICU's word.txt)
#
#   DESCRIPTION:
#       Not a port: replaces ICU's word break iterator. The rules are UAX #29's word boundary
#       rules for Unicode 17 as ICU 78's word.txt states them, including ICU's tailorings:
#       complex-context (South East Asian) letters count as letters, and runs of Han,
#       Hiragana and Katakana stay together (ICU then subdivides such runs with dictionaries,
#       which this port does not have). Each boundary carries ICU's rule status of the word
#       before it: UBRK_WORD_NUMBER (100), UBRK_WORD_LETTER (200), UBRK_WORD_IDEO (400) or
#       UBRK_WORD_NONE (0).
#
#==============================================================================================

# mark: Classes =================================================================================

# UWordBreakValues, and two classes of ICU's word.txt that are not Word_Break values.
let wb_other: u32 = 0
let wb_aletter: u32 = 1
let wb_format: u32 = 2
let wb_katakana: u32 = 3
let wb_mid_letter: u32 = 4
let wb_mid_num: u32 = 5
let wb_numeric: u32 = 6
let wb_extend_num_let: u32 = 7
let wb_cr: u32 = 8
let wb_extend: u32 = 9
let wb_lf: u32 = 10
let wb_mid_num_let: u32 = 11
let wb_newline: u32 = 12
let wb_regional_indicator: u32 = 13
let wb_hebrew_letter: u32 = 14
let wb_single_quote: u32 = 15
let wb_double_quote: u32 = 16
let wb_zwj: u32 = 21
let wb_wseg_space: u32 = 22
## ICU's $Han and $Hiragana: part of $KanaKanji with $Katakana.
let wb_kanji_hiragana: u32 = 30
## ICU's $HangulSyllable (U+AC00..U+D7A3): not part of $ALetterPlus in ICU's rules.
let wb_hangul_syllable: u32 = 31

# The rule statuses of ubrk.h.
let ubrk_word_none: u32 = 0
let ubrk_word_number: u32 = 100
let ubrk_word_letter: u32 = 200
let ubrk_word_kana: u32 = 300
let ubrk_word_ideo: u32 = 400

# UScriptCode values of word.txt's $Han and $Hiragana.
let usc_han: u32 = 17
let usc_hiragana: u32 = 20

# ULineBreak U_LB_COMPLEX_CONTEXT, and UGraphemeClusterBreak Control.
let u_lb_complex_context: u32 = 24

## The class of `code_point` in ICU's word rules.
func word_class(code_point: u32) -> u32:
    let record = ucd_record(code_point)
    let wb = ucd_hex(ucd_records, record + ucd_field_word_break, 2)
    if wb == wb_extend or wb == wb_format or wb == wb_zwj or wb == wb_katakana:
        return wb
    if code_point >= 0xac00 and code_point <= 0xd7a3:
        return wb_hangul_syllable
    let script = ucd_hex(ucd_records, record + ucd_field_script, 2)
    if script == usc_han or script == usc_hiragana:
        if wb == wb_other or wb == wb_aletter:
            return wb_kanji_hiragana
    # $ALetterPlus = [$ALetter - $dictionaryCJK [$ComplexContext - $Extend - $Control]]
    if ucd_hex(ucd_records, record + ucd_field_line_break, 2) == u_lb_complex_context:
        if ucd_hex(ucd_records, record + ucd_field_grapheme_cluster_break, 2) != gcb_control:
            return wb_aletter
    return wb

func is_word_ignorable(class: u32) -> bool:
    return class == wb_extend or class == wb_format or class == wb_zwj

func is_ahletter(class: u32) -> bool:
    return class == wb_aletter or class == wb_hebrew_letter

func is_mid_num_let_q(class: u32) -> bool:
    return class == wb_mid_num_let or class == wb_single_quote

func is_kana_kanji(class: u32) -> bool:
    return class == wb_katakana or class == wb_kanji_hiragana

## The rule status of a word made of the one character of class `class`.
func word_single_status(class: u32) -> u32:
    if is_ahletter(class) or class == wb_hangul_syllable:
        return ubrk_word_letter
    if class == wb_numeric:
        return ubrk_word_number
    if is_kana_kanji(class):
        return ubrk_word_ideo
    return ubrk_word_none

# mark: Boundaries ==============================================================================

## Append the word boundaries of `code_points` to `boundaries`, and the rule status of each
## boundary to `statuses` (the status of the first boundary is 0).
func word_boundaries(code_points: const u32[], boundaries: CodePointBuffer*, statuses: CodePointBuffer*):
    boundaries.append(0)
    statuses.append(ubrk_word_none)
    let count = code_points.length
    if count == 0:
        return
    var classes = CodePointBuffer.create(count)
    for code_point in code_points:
        classes.append(word_class(code_point))
    let class = classes.view()
    # WB4: base[i] is the character an ignorable character i attaches to (itself otherwise);
    # an ignorable at the start of text or after CR, LF or Newline is its own base.
    var base = CodePointBuffer.create(count)
    for index in 0..<count:
        var own = true
        if index > 0 and is_word_ignorable(class[index]):
            let previous = base.items[index - 1]
            let previous_class = class[(usize)previous]
            if previous_class != wb_cr and previous_class != wb_lf and previous_class != wb_newline:
                own = false
                base.append(previous)
        if own:
            base.append((u32)index)
    # next_base[i]: the first non-ignorable character at or after i, or `count`.
    var next_base = CodePointBuffer.create(count + 1)
    for _unused in 0..=count:
        next_base.append((u32)count)
    var index = count
    while index > 0:
        index -= 1
        next_base.items[index] = (u32)index if not is_word_ignorable(class[index]) else next_base.items[index + 1]

    var status = word_single_status(class[0])
    var status_ends_without_extension = false
    # The regional indicators ending the text before the position, counted over bases.
    var regional_indicators: usize = 1 if class[0] == wb_regional_indicator else 0
    for position in 1..<count:
        let current = class[position]
        let raw_previous = class[position - 1]
        let previous_index = (usize)base.items[position - 1]
        let previous = class[previous_index]
        var joined = false
        var tag = status
        # WB3: CR × LF
        if raw_previous == wb_cr and current == wb_lf:
            joined = true
            tag = ubrk_word_none
        # WB3a, WB3b: break after and before newlines
        elif raw_previous == wb_cr or raw_previous == wb_lf or raw_previous == wb_newline:
            joined = false
        elif current == wb_cr or current == wb_lf or current == wb_newline:
            joined = false
        # WB3c: ZWJ × \p{Extended_Pictographic}
        elif raw_previous == wb_zwj and ucd_has_binary_property(code_points[position], uchar_extended_pictographic):
            joined = true
            tag = ubrk_word_none
        # WB3d: WSegSpace × WSegSpace
        elif raw_previous == wb_wseg_space and current == wb_wseg_space:
            joined = true
            tag = ubrk_word_none
        # WB4: X (Extend | Format | ZWJ)* → X. ICU's rule for Hangul syllables has no
        # $ExFm*: the ignorables attach by the untagged rule 4, whose status is 0.
        elif is_word_ignorable(current):
            joined = true
            # So do those of ICU's rules 7a and 13a, which end in a Single_Quote or
            # ExtendNumLet without $ExFm*.
            if previous == wb_hangul_syllable or (status_ends_without_extension and not is_word_ignorable(raw_previous)):
                tag = ubrk_word_none
        # ICU's $HangulSyllable $HangulSyllable {200}, with nothing between them.
        elif current == wb_hangul_syllable and raw_previous == wb_hangul_syllable:
            joined = true
            tag = ubrk_word_letter
        else:
            let rule = word_join_rule(class, base.view(), next_base.view(), previous_index, previous, position, current, regional_indicators)
            if rule != 0xffffffff:
                joined = true
                tag = rule
        if joined:
            if not is_word_ignorable(current):
                # Rules 7a (Hebrew_Letter × Single_Quote) and 13a (× ExtendNumLet).
                status_ends_without_extension = current == wb_extend_num_let or (current == wb_single_quote and previous == wb_hebrew_letter)
            status = tag
        else:
            boundaries.append((u32)position)
            statuses.append(status)
            status = word_single_status(current) if not is_word_ignorable(current) else ubrk_word_none
            status_ends_without_extension = false
        if not is_word_ignorable(current):
            regional_indicators = regional_indicators + 1 if current == wb_regional_indicator else 0
    boundaries.append((u32)count)
    statuses.append(status)

## Whether rules WB5 through WB16 keep the characters of classes `previous` (at
## `previous_index`, a base) and `current` (at `position`) together; the rule's status tag
## if so, else 0xffffffff.
func word_join_rule(class: const u32[], base: const u32[], next_base: const u32[], previous_index: usize, previous: u32, position: usize, current: u32, regional_indicators: usize) -> u32:
    let count = class.length
    let next_index = (usize)next_base[position + 1] if position + 1 < count else count
    let next = class[next_index] if next_index < count else wb_other
    let before_index: usize? = (usize)base[previous_index - 1] if previous_index > 0 else none
    var before = wb_other
    if let found = before_index:
        before = class[found]
    # WB5: AHLetter × AHLetter
    if is_ahletter(previous) and is_ahletter(current):
        return ubrk_word_letter
    # WB6: AHLetter × (MidLetter | MidNumLetQ) AHLetter
    if is_ahletter(previous) and (current == wb_mid_letter or is_mid_num_let_q(current)) and is_ahletter(next):
        return ubrk_word_letter
    # WB7: AHLetter (MidLetter | MidNumLetQ) × AHLetter
    if is_ahletter(before) and (previous == wb_mid_letter or is_mid_num_let_q(previous)) and is_ahletter(current):
        return ubrk_word_letter
    # WB7a: Hebrew_Letter × Single_Quote
    if previous == wb_hebrew_letter and current == wb_single_quote:
        return ubrk_word_letter
    # WB7b: Hebrew_Letter × Double_Quote Hebrew_Letter
    if previous == wb_hebrew_letter and current == wb_double_quote and next == wb_hebrew_letter:
        return ubrk_word_letter
    # WB7c: Hebrew_Letter Double_Quote × Hebrew_Letter
    if before == wb_hebrew_letter and previous == wb_double_quote and current == wb_hebrew_letter:
        return ubrk_word_letter
    # WB8: Numeric × Numeric
    if previous == wb_numeric and current == wb_numeric:
        return ubrk_word_number
    # WB9: AHLetter × Numeric (ICU's rule status is the number's)
    if is_ahletter(previous) and current == wb_numeric:
        return ubrk_word_number
    # WB10: Numeric × AHLetter
    if previous == wb_numeric and is_ahletter(current):
        return ubrk_word_letter
    # WB11: Numeric (MidNum | MidNumLetQ) × Numeric
    if before == wb_numeric and (previous == wb_mid_num or is_mid_num_let_q(previous)) and current == wb_numeric:
        return ubrk_word_number
    # WB12: Numeric × (MidNum | MidNumLetQ) Numeric
    if previous == wb_numeric and (current == wb_mid_num or is_mid_num_let_q(current)) and next == wb_numeric:
        return ubrk_word_number
    # WB13 and ICU's $KanaKanji $KanaKanji: Katakana × Katakana, and runs of Han, Hiragana
    # and Katakana (kept for dictionary segmentation in ICU)
    if is_kana_kanji(previous) and is_kana_kanji(current):
        return ubrk_word_ideo
    # WB13a: (AHLetter | Numeric | Katakana | ExtendNumLet) × ExtendNumLet
    if current == wb_extend_num_let:
        if is_ahletter(previous) or previous == wb_extend_num_let:
            return ubrk_word_letter
        if previous == wb_numeric:
            return ubrk_word_number
        if previous == wb_katakana:
            return ubrk_word_ideo
    # WB13b: ExtendNumLet × (AHLetter | Numeric | Katakana)
    if previous == wb_extend_num_let:
        if is_ahletter(current):
            return ubrk_word_letter
        if current == wb_numeric:
            return ubrk_word_number
        if current == wb_katakana:
            return ubrk_word_ideo
    # WB15, WB16: pairs of regional indicators
    if previous == wb_regional_indicator and current == wb_regional_indicator and regional_indicators % 2 == 1:
        return ubrk_word_none
    # WB999: Any ÷ Any
    return 0xffffffff