Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.0

0.6.0 / src / web_unicode / break_line.lucb

#==============================================================================================
#
#   break_line - line break opportunities (UAX #14)
#
#   DESCRIPTION:
#       Not a port: replaces ICU's line break iterator (ICU 78's line.txt: the UAX #14 rules
#       of Unicode 17 with CJ resolved to NS, as CSS line-break: strict). ICU splits runs of
#       complex-context scripts (Thai, Lao, Khmer, Myanmar) with dictionaries; this port has
#       none and resolves them as LB1 says.
#
#       The rules after LB9 see the text as groups: a character with the combining marks and
#       joiners attached to it by LB9. Each rule is checked at a group boundary in the order
#       of UAX #14, looking at the groups before and after it.
#
#==============================================================================================

# mark: Classes =================================================================================

# ULineBreak values.
let lb_xx: u32 = 0
let lb_ai: u32 = 1
let lb_al: u32 = 2
let lb_b2: u32 = 3
let lb_ba: u32 = 4
let lb_bb: u32 = 5
let lb_bk: u32 = 6
let lb_cb: u32 = 7
let lb_cl: u32 = 8
let lb_cm: u32 = 9
let lb_cr: u32 = 10
let lb_ex: u32 = 11
let lb_gl: u32 = 12
let lb_hy: u32 = 13
let lb_id: u32 = 14
let lb_in: u32 = 15
let lb_is: u32 = 16
let lb_lf: u32 = 17
let lb_ns: u32 = 18
let lb_nu: u32 = 19
let lb_op: u32 = 20
let lb_po: u32 = 21
let lb_pr: u32 = 22
let lb_qu: u32 = 23
let lb_sa: u32 = 24
let lb_sg: u32 = 25
let lb_sp: u32 = 26
let lb_sy: u32 = 27
let lb_zw: u32 = 28
let lb_nl: u32 = 29
let lb_wj: u32 = 30
let lb_h2: u32 = 31
let lb_h3: u32 = 32
let lb_jl: u32 = 33
let lb_jt: u32 = 34
let lb_jv: u32 = 35
let lb_cp: u32 = 36
let lb_cj: u32 = 37
let lb_hl: u32 = 38
let lb_ri: u32 = 39
let lb_eb: u32 = 40
let lb_em: u32 = 41
let lb_zwj: u32 = 42
let lb_ak: u32 = 43
let lb_ap: u32 = 44
let lb_as: u32 = 45
let lb_vf: u32 = 46
let lb_vi: u32 = 47
let lb_hh: u32 = 48

# UEastAsianWidth values of LB19a's and LB30's $EastAsian: F, W and H.
let ea_halfwidth: u32 = 2
let ea_fullwidth: u32 = 3
let ea_wide: u32 = 5

# UCharCategory values of the quotation marks.
let u_initial_punctuation: u32 = 28
let u_final_punctuation: u32 = 29

## A group of LB9: a character and the CM and ZWJ characters attached to it.
struct LineGroup:
    ## The index of the group's first code point.
    var start: u32
    ## The group's class after LB1 and LB10.
    var class: u32
    ## Whether the last code point of the group is a ZWJ (LB8a).
    var ends_with_zwj: bool
    ## Properties of the group's first code point.
    var east_asian: bool
    var initial_quote: bool
    var final_quote: bool
    var unassigned_pictographic: bool
    var dotted_circle: bool

## LB1: the resolved line break class of `code_point`.
func line_break_class_resolved(code_point: u32) -> u32:
    let record = ucd_record(code_point)
    let lb = ucd_hex(ucd_records, record + ucd_field_line_break, 2)
    if lb == lb_ai or lb == lb_sg or lb == lb_xx:
        return lb_al
    if lb == lb_sa:
        let gc = ucd_hex(ucd_records, record + ucd_field_general_category, 2)
        return lb_cm if gc == u_non_spacing_mark or gc == u_combining_spacing_mark else lb_al
    if lb == lb_cj:
        return lb_ns
    return lb

## Append the line break opportunities of `code_points` to `boundaries`.
func line_boundaries(code_points: const u32[], boundaries: CodePointBuffer*):
    boundaries.append(0)
    let count = code_points.length
    if count == 0:
        return
    # LB9 and LB10: build the groups.
    var groups = new LineGroup[count] else trap("out of memory")
    var group_count: usize = 0
    for index in 0..<count:
        let code_point = code_points[index]
        let class = line_break_class_resolved(code_point)
        if (class == lb_cm or class == lb_zwj) and group_count > 0:
            let base_raw = line_break_class_resolved(code_points[(usize)groups[group_count - 1].start])
            if base_raw != lb_bk and base_raw != lb_cr and base_raw != lb_lf and base_raw != lb_nl and base_raw != lb_sp and base_raw != lb_zw:
                groups[group_count - 1].ends_with_zwj = class == lb_zwj
                continue
        let record = ucd_record(code_point)
        let ea = ucd_hex(ucd_records, record + ucd_field_east_asian_width, 2)
        let gc = ucd_hex(ucd_records, record + ucd_field_general_category, 2)
        groups[group_count] = LineGroup(start = (u32)index,
                                        class = lb_al if class == lb_cm or class == lb_zwj else class,
                                        ends_with_zwj = class == lb_zwj,
                                        east_asian = ea == ea_fullwidth or ea == ea_wide or ea == ea_halfwidth,
                                        initial_quote = gc == u_initial_punctuation, final_quote = gc == u_final_punctuation,
                                        unassigned_pictographic = gc == u_unassigned and ucd_has_binary_property(code_point, uchar_extended_pictographic),
                                        dotted_circle = code_point == 0x25cc)
        group_count += 1
    let group = groups[..<group_count]
    var regional_indicators: usize = 0
    for k in 1..<group_count:
        regional_indicators = regional_indicators + 1 if group[k - 1].class == lb_ri else 0
        if line_break_before_group(group, k, regional_indicators):
            boundaries.append(group[k].start)
    boundaries.append((u32)count)

## The class of group `index`, or none outside the text.
func group_class(group: const LineGroup[], index: isize) -> u32?:
    if index < 0 or index >= (isize)group.length:
        return none
    return group[(usize)index].class

## The index of the first group before `index` that is not SP, or -1.
func skip_spaces_back(group: const LineGroup[], index: isize) -> isize:
    var position = index
    while position >= 0 and group[(usize)position].class == lb_sp:
        position -= 1
    return position

func is_ak_dotted_as(group: const LineGroup*) -> bool:
    return group.class == lb_ak or group.class == lb_as or group.dotted_circle

func is_ak_dotted(group: const LineGroup*) -> bool:
    return group.class == lb_ak or group.dotted_circle

## Whether there is a line break opportunity between groups k - 1 and k.
func line_break_before_group(group: const LineGroup[], k: usize, regional_indicators: usize) -> bool:
    let before = &group[k - 1]
    let after = &group[k]
    let b = before.class
    let a = after.class
    let next: u32? = group_class(group, (isize)k + 1)
    let before_before: u32? = group_class(group, (isize)k - 2)
    # LB4: BK !
    if b == lb_bk:
        return true
    # LB5: CR × LF; CR !; LF !; NL !
    if b == lb_cr and a == lb_lf:
        return false
    if b == lb_cr or b == lb_lf or b == lb_nl:
        return true
    # LB6: × (BK | CR | LF | NL)
    if a == lb_bk or a == lb_cr or a == lb_lf or a == lb_nl:
        return false
    # LB7: × SP; × ZW
    if a == lb_sp or a == lb_zw:
        return false
    # LB8: ZW SP* ÷
    let before_spaces = skip_spaces_back(group, (isize)k - 1)
    if before_spaces >= 0 and group[(usize)before_spaces].class == lb_zw:
        return true
    # LB8a: ZWJ ×
    if before.ends_with_zwj:
        return false
    # LB11: × WJ; WJ ×
    if a == lb_wj or b == lb_wj:
        return false
    # LB12: GL ×
    if b == lb_gl:
        return false
    # LB12a: [^SP BA HY HH] × GL
    if a == lb_gl and b != lb_sp and b != lb_ba and b != lb_hy and b != lb_hh:
        return false
    # LB13: × CL; × CP; × EX; × SY
    if a == lb_cl or a == lb_cp or a == lb_ex or a == lb_sy:
        return false
    # LB14: OP SP* ×
    if before_spaces >= 0 and group[(usize)before_spaces].class == lb_op:
        return false
    # LB15a: (sot | BK | CR | LF | NL | OP | QU | GL | SP | ZW) [\p{Pi}&QU] SP* ×
    if before_spaces >= 0:
        let quote = &group[(usize)before_spaces]
        if quote.class == lb_qu and quote.initial_quote:
            let preceding: u32? = group_class(group, before_spaces - 1)
            if preceding == none:
                return false
            let value = preceding else return false
            if value == lb_bk or value == lb_cr or value == lb_lf or value == lb_nl or value == lb_op or value == lb_qu or value == lb_gl or value == lb_sp or value == lb_zw:
                return false
    # LB15b: × [\p{Pf}&QU] (SP | GL | WJ | CL | QU | CP | EX | IS | SY | BK | CR | LF | NL | ZW | eot)
    if a == lb_qu and after.final_quote:
        let following = next else return false
        if following == lb_sp or following == lb_gl or following == lb_wj or following == lb_cl or following == lb_qu or following == lb_cp or following == lb_ex or following == lb_is or following == lb_sy or following == lb_bk or following == lb_cr or following == lb_lf or following == lb_nl or following == lb_zw:
            return false
    # LB15c: SP ÷ IS NU
    if b == lb_sp and a == lb_is and next == lb_nu:
        return true
    # LB15d: × IS
    if a == lb_is:
        return false
    # LB16: (CL | CP) SP* × NS
    if a == lb_ns and before_spaces >= 0 and (group[(usize)before_spaces].class == lb_cl or group[(usize)before_spaces].class == lb_cp):
        return false
    # LB17: B2 SP* × B2
    if a == lb_b2 and before_spaces >= 0 and group[(usize)before_spaces].class == lb_b2:
        return false
    # LB18: SP ÷
    if b == lb_sp:
        return true
    # LB19: × [QU - \p{Pi}]; [QU - \p{Pf}] ×
    if a == lb_qu and not after.initial_quote:
        return false
    if b == lb_qu and not before.final_quote:
        return false
    # LB19a: [^$EastAsian] × QU; × QU ( [^$EastAsian] | eot ); QU × [^$EastAsian];
    #        ( sot | [^$EastAsian] ) QU ×
    if a == lb_qu:
        if not before.east_asian:
            return false
        if k + 1 >= group.length or not group[k + 1].east_asian:
            return false
    if b == lb_qu:
        if not after.east_asian:
            return false
        if k < 2 or not group[k - 2].east_asian:
            return false
    # LB20: ÷ CB; CB ÷
    if a == lb_cb or b == lb_cb:
        return true
    # LB20a: ( sot | BK | CR | LF | NL | SP | ZW | CB | GL ) ( HY | HH ) × ( AL | HL )
    if (b == lb_hy or b == lb_hh) and (a == lb_al or a == lb_hl):
        let preceding = before_before else return false
        if preceding == lb_bk or preceding == lb_cr or preceding == lb_lf or preceding == lb_nl or preceding == lb_sp or preceding == lb_zw or preceding == lb_cb or preceding == lb_gl:
            return false
    # LB21: × BA; × HH; × HY; × NS; BB ×
    if a == lb_ba or a == lb_hh or a == lb_hy or a == lb_ns or b == lb_bb:
        return false
    # LB21a: HL (HY | HH) × [^HL]
    if (b == lb_hy or b == lb_hh) and before_before == lb_hl and a != lb_hl:
        return false
    # LB21b: SY × HL
    if b == lb_sy and a == lb_hl:
        return false
    # LB22: × IN
    if a == lb_in:
        return false
    # LB23: (AL | HL) × NU; NU × (AL | HL)
    if (b == lb_al or b == lb_hl) and a == lb_nu:
        return false
    if b == lb_nu and (a == lb_al or a == lb_hl):
        return false
    # LB23a: PR × (ID | EB | EM); (ID | EB | EM) × PO
    if b == lb_pr and (a == lb_id or a == lb_eb or a == lb_em):
        return false
    if (b == lb_id or b == lb_eb or b == lb_em) and a == lb_po:
        return false
    # LB24: (PR | PO) × (AL | HL); (AL | HL) × (PR | PO)
    if (b == lb_pr or b == lb_po) and (a == lb_al or a == lb_hl):
        return false
    if (b == lb_al or b == lb_hl) and (a == lb_pr or a == lb_po):
        return false
    # LB25: do not break numbers
    if line_number_join(group, k):
        return false
    # LB26: JL × (JL | JV | H2 | H3); (JV | H2) × (JV | JT); (JT | H3) × JT
    if b == lb_jl and (a == lb_jl or a == lb_jv or a == lb_h2 or a == lb_h3):
        return false
    if (b == lb_jv or b == lb_h2) and (a == lb_jv or a == lb_jt):
        return false
    if (b == lb_jt or b == lb_h3) and a == lb_jt:
        return false
    # LB27: (JL | JV | JT | H2 | H3) × PO; PR × (JL | JV | JT | H2 | H3)
    let a_korean = a == lb_jl or a == lb_jv or a == lb_jt or a == lb_h2 or a == lb_h3
    let b_korean = b == lb_jl or b == lb_jv or b == lb_jt or b == lb_h2 or b == lb_h3
    if b_korean and a == lb_po:
        return false
    if b == lb_pr and a_korean:
        return false
    # LB28: (AL | HL) × (AL | HL)
    if (b == lb_al or b == lb_hl) and (a == lb_al or a == lb_hl):
        return false
    # LB28a: AP × (AK | ◌ | AS); (AK | ◌ | AS) × (VF | VI); (AK | ◌ | AS) VI × (AK | ◌);
    #        (AK | ◌ | AS) × (AK | ◌ | AS) VF
    if b == lb_ap and is_ak_dotted_as(after):
        return false
    if is_ak_dotted_as(before) and (a == lb_vf or a == lb_vi):
        return false
    if b == lb_vi and k >= 2 and is_ak_dotted_as(&group[k - 2]) and is_ak_dotted(after):
        return false
    if is_ak_dotted_as(before) and is_ak_dotted_as(after) and next == lb_vf:
        return false
    # LB29: IS × (AL | HL)
    if b == lb_is and (a == lb_al or a == lb_hl):
        return false
    # LB30: (AL | HL | NU) × [OP-$EastAsian]; [CP-$EastAsian] × (AL | HL | NU)
    if (b == lb_al or b == lb_hl or b == lb_nu) and a == lb_op and not after.east_asian:
        return false
    if b == lb_cp and not before.east_asian and (a == lb_al or a == lb_hl or a == lb_nu):
        return false
    # LB30a: sot (RI RI)* RI × RI; [^RI] (RI RI)* RI × RI
    if b == lb_ri and a == lb_ri and regional_indicators % 2 == 1:
        return false
    # LB30b: EB × EM; [\p{Extended_Pictographic}&\p{Cn}] × EM
    if a == lb_em and (b == lb_eb or before.unassigned_pictographic):
        return false
    # LB31: ÷
    return true

## LB25: whether the groups k - 1 and k are inside one number of the form
## ((PR | PO) (OP | HY)? IS? NU (NU | SY | IS)* (CL | CP)? (PR | PO)?), which UAX #14 states as
##     (PR | PO) × ( OP | HY )? IS? NU
##     ( OP | HY ) × IS? NU
##     IS × NU
##     NU (NU | SY | IS)* × (NU | SY | IS | CL | CP)
##     NU (NU | SY | IS)* (CL | CP)? × (PO | PR)
func line_number_join(group: const LineGroup[], k: usize) -> bool:
    let b = group[k - 1].class
    let a = group[k].class
    let next: u32? = group_class(group, (isize)k + 1)
    let after_next: u32? = group_class(group, (isize)k + 2)
    # (PR | PO) × ( OP | HY )? IS? NU
    if b == lb_pr or b == lb_po:
        if a == lb_nu:
            return true
        if (a == lb_op or a == lb_hy) and (next == lb_nu or (next == lb_is and after_next == lb_nu)):
            return true
        if a == lb_is and next == lb_nu:
            return true
    # ( OP | HY ) × IS? NU
    if b == lb_op or b == lb_hy:
        if a == lb_nu or (a == lb_is and next == lb_nu):
            return true
    # IS × NU
    if b == lb_is and a == lb_nu:
        return true
    # NU (NU | SY | IS)* × (NU | SY | IS | CL | CP)
    if a == lb_nu or a == lb_sy or a == lb_is or a == lb_cl or a == lb_cp:
        if number_run_ends_at(group, (isize)k - 1):
            return true
    # NU (NU | SY | IS)* (CL | CP)? × (PO | PR)
    if a == lb_po or a == lb_pr:
        if number_run_ends_at(group, (isize)k - 1):
            return true
        if (b == lb_cl or b == lb_cp) and number_run_ends_at(group, (isize)k - 2):
            return true
    return false

## Whether the groups ending at `index` are NU (NU | SY | IS)*.
func number_run_ends_at(group: const LineGroup[], index: isize) -> bool:
    var position = index
    while position >= 0:
        let class = group[(usize)position].class
        if class == lb_nu:
            return true
        if class != lb_sy and class != lb_is:
            return false
        position -= 1
    return false