Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-std

Source of release 0.7.4

0.7.4 / tests / unicode_graphemes / check.lucb

import c
import files
import memory
import unicode
import utf8

pub func main(arguments: c.str[]) -> i32!:
    let corpus_file = try files.read(arguments[1])
    defer corpus_file.release()
    let corpus = corpus_file.value
    var empty: u8[]
    var refusing = memory.FixedBuffer.over(empty)
    with refusing:
        var position: usize = 0
        var cases: usize = 0
        while position < corpus.length:
            let length = (usize)memory.read[u32](corpus.data + position)
            let count = (usize)memory.read[u32](corpus.data + position + 4)
            position += 8
            let text = (str)corpus[position..<position + length]
            position += length
            let lengths = corpus[position..<position + count * 4]
            position += count * 4
            var iterator = try unicode.GraphemeIterator.over(text)
            assert(iterator.byte_offset() == 0)
            var offset: usize = 0
            for index in 0..<count:
                let expected = (usize)memory.read[u32](lengths.data + index * 4)
                let cluster = iterator.next() else trap("a grapheme cluster is missing")
                if cluster.length != expected:
                    print(f"grapheme case {cases}, cluster {index}: {cluster.length} bytes, expected {expected}")
                    trap("a grapheme boundary differs from the reference")
                assert(cluster.bytes.data == text.bytes.data + offset)
                assert(cluster == (str)text.bytes[offset..<offset + expected])
                offset += expected
                assert(iterator.byte_offset() == offset)
            assert(offset == length)
            assert(iterator.next() == none and iterator.next() == none)
            cases += 1
        assert(cases > 700)
        var zero: unicode.GraphemeIterator
        assert(zero.next() == none and zero.byte_offset() == 0)
        var byte: u8[1]
        for value in 128..<256:
            byte[0] = (u8)value
            var rejected = false
            _ = unicode.GraphemeIterator.over((str)byte) catch failure:
                assert(failure.code == utf8.invalid_sequence)
                rejected = true
                recover unicode.GraphemeIterator()
            assert(rejected)
        var malformed: u8[4] = [237, 160, 128, 65]
        var rejected = false
        _ = unicode.GraphemeIterator.over((str)malformed) catch failure:
            assert(failure.code == utf8.invalid_sequence)
            rejected = true
            recover unicode.GraphemeIterator()
        assert(rejected)
    print("ok Unicode extended grapheme boundaries")
    return 0