Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-browser-foundation

Source of release 0.6.1

0.6.1 / src / web_unicode / break_iterator.lucb

#==============================================================================================
#
#   break_iterator - the navigation of ICU's BreakIterator over precomputed boundaries
#
#   DESCRIPTION:
#       Not a port: replaces icu::BreakIterator (RuleBasedBreakIterator) for the Segmenter.
#       Setting the text computes every boundary at once (ICU computes them lazily through a
#       cache; the boundaries are the same). first, next, current, following, preceding and
#       isBoundary keep ICU's semantics, including where they leave the current position.
#       Offsets are code units of the text: bytes for UTF-8 text, code units for UTF-16.
#
#==============================================================================================

## BreakIterator::DONE.
let break_iterator_done: i32 = -1

## A break iterator of one granularity over one text.
struct BreakIterator:
    var granularity: SegmenterGranularity
    ## The boundaries, ascending, in code units.
    var boundaries: CodePointBuffer
    ## The word rule status of each boundary (word iterators only).
    var statuses: CodePointBuffer
    ## The code unit offset of each code point, then the length (for alignment).
    var starts: CodePointBuffer
    ## The index in `boundaries` of the current boundary.
    var position: usize

## BreakIterator::create*Instance: an iterator over the empty text.
func break_iterator_create(granularity: SegmenterGranularity) -> BreakIterator*:
    var starts = CodePointBuffer.create(1)
    starts.append(0)
    var boundaries = CodePointBuffer.create(1)
    boundaries.append(0)
    var statuses = CodePointBuffer.create(1)
    statuses.append(0)
    return new BreakIterator(granularity = granularity, boundaries = boundaries, statuses = statuses, starts = starts, position = 0) else trap("out of memory")

## BreakIterator::clone. The boundary arrays are never changed once computed, so the clone
## may share them.
func break_iterator_clone(this: const BreakIterator*) -> BreakIterator*:
    return new BreakIterator(granularity = this.granularity, boundaries = this.boundaries, statuses = this.statuses, starts = this.starts, position = this.position) else trap("out of memory")

## BreakIterator::setText over UTF-8 bytes.
func break_iterator_set_text_utf8(this: BreakIterator*, bytes: const u8[]):
    var code_points = CodePointBuffer.create(bytes.length + 1)
    var starts = CodePointBuffer.create(bytes.length + 2)
    decode_utf8(bytes, &code_points, &starts)
    break_iterator_set_code_points(this, code_points.view(), starts)

## BreakIterator::setText over UTF-16 code units.
func break_iterator_set_text_utf16(this: BreakIterator*, units: const u16[]):
    var code_points = CodePointBuffer.create(units.length + 1)
    var starts = CodePointBuffer.create(units.length + 2)
    decode_utf16(units, &code_points, &starts)
    break_iterator_set_code_points(this, code_points.view(), starts)

## Compute the boundaries of `code_points`, whose code unit offsets are `starts`.
func break_iterator_set_code_points(this: BreakIterator*, code_points: const u32[], starts: CodePointBuffer):
    var indexes = CodePointBuffer.create(code_points.length + 2)
    var statuses = CodePointBuffer.create(1)
    match this.granularity:
        .grapheme: grapheme_boundaries(code_points, &indexes)
        .word: word_boundaries(code_points, &indexes, &statuses)
        .line: line_boundaries(code_points, &indexes)
        .sentence: sentence_boundaries(code_points, &indexes)
        _: trap("VERIFY_NOT_REACHED")
    # Code point indexes to code units.
    var boundaries = CodePointBuffer.create(indexes.count)
    for index in indexes.view():
        boundaries.append(starts.items[(usize)index])
    this.boundaries = boundaries
    this.statuses = statuses
    this.starts = starts
    this.position = 0

## The length of the text in code units.
func break_iterator_length(this: const BreakIterator*) -> u32:
    return this.starts.items[this.starts.count - 1]

## The start of the code point containing `offset`, clamped to the text
## (utext_setNativeIndex + utext_getNativeIndex).
func break_iterator_align(this: const BreakIterator*, offset: usize) -> u32:
    let length = break_iterator_length(this)
    if offset >= (usize)length:
        return length
    # The last code point start not after `offset`.
    var low: usize = 0
    var high = this.starts.count - 1
    while low + 1 < high:
        let middle = low + (high - low) // 2
        if (usize)this.starts.items[middle] <= offset:
            low = middle
        else:
            high = middle
    return this.starts.items[low]

## The index of the last boundary not after `offset` (BreakCache::seek).
func break_iterator_seek(this: const BreakIterator*, offset: u32) -> usize:
    var low: usize = 0
    var high = this.boundaries.count
    while low + 1 < high:
        let middle = low + (high - low) // 2
        if this.boundaries.items[middle] <= offset:
            low = middle
        else:
            high = middle
    return low

## BreakIterator::first.
func break_iterator_first(this: BreakIterator*) -> i32:
    this.position = 0
    return (i32)this.boundaries.items[0]

## BreakIterator::next.
func break_iterator_next(this: BreakIterator*) -> i32:
    if this.position + 1 >= this.boundaries.count:
        return break_iterator_done
    this.position += 1
    return (i32)this.boundaries.items[this.position]

## BreakIterator::current.
func break_iterator_current(this: const BreakIterator*) -> i32:
    return (i32)this.boundaries.items[this.position]

## BreakIterator::following: the first boundary after `offset`, or DONE.
func break_iterator_following(this: BreakIterator*, offset: i32) -> i32:
    if offset < 0:
        return break_iterator_first(this)
    let aligned = break_iterator_align(this, (usize)offset)
    this.position = break_iterator_seek(this, aligned)
    return break_iterator_next(this)

## BreakIterator::preceding: the last boundary before `offset`, or DONE.
func break_iterator_preceding(this: BreakIterator*, offset: i32) -> i32:
    if offset < 0 or offset > (i32)break_iterator_length(this):
        this.position = this.boundaries.count - 1
        if offset < 0:
            this.position = 0
            return break_iterator_done
        return break_iterator_current(this)
    let aligned = break_iterator_align(this, (usize)offset)
    let index = break_iterator_seek(this, aligned)
    if this.boundaries.items[index] == aligned:
        if index == 0:
            this.position = 0
            return break_iterator_done
        this.position = index - 1
    else:
        this.position = index
    return break_iterator_current(this)

## BreakIterator::isBoundary: whether `offset` is a boundary. Otherwise the current position
## moves to the following boundary.
func break_iterator_is_boundary(this: BreakIterator*, offset: i32) -> bool:
    if offset < 0:
        _ = break_iterator_first(this)
        return false
    let aligned = break_iterator_align(this, (usize)offset)
    this.position = break_iterator_seek(this, aligned)
    let result = (i32)this.boundaries.items[this.position] == offset
    if result and (i32)aligned < offset:
        # The offset is beyond the end of the text; the position stays at its end.
        return false
    if not result:
        _ = break_iterator_next(this)
    return result

## BreakIterator::getRuleStatus: the status of the current boundary (word iterators).
func break_iterator_rule_status(this: const BreakIterator*) -> i32:
    if this.position >= this.statuses.count:
        return 0
    return (i32)this.statuses.items[this.position]