Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-std

Source of release 0.7.2

0.7.2 / tests / unicode_normalization / check.lucb

import c
import files
import interop
import memory
import strings
import unicode
import utf8

pub func main(arguments: c.str[]) -> i32!:
    let corpus_file = try files.read(arguments[1])
    defer corpus_file.release()
    let corpus = corpus_file.value
    var position: usize = 0
    var count: usize = 0
    while position < corpus.length:
        let form = (unicode.NormalizationForm)(u8)memory.read[u32](corpus.data + position)
        let input_length = (usize)memory.read[u32](corpus.data + position + 4)
        let output_length = (usize)memory.read[u32](corpus.data + position + 8)
        position += 12
        let input = (str)corpus[position..<position + input_length]
        position += input_length
        let expected = (str)corpus[position..<position + output_length]
        position += output_length
        let normalized = try unicode.normalize(input, form, expected.length)
        defer normalized.release()
        let actual = normalized.value
        if actual != expected:
            print(f"normalization record {count}, form {(u8)form}")
            trap("Unicode normalization differs from the reference")
        assert(actual.bytes.data[actual.length] == 0)
        count += 1
    assert(count > 100000)
    let contracted = try unicode.normalize("fi", unicode.NormalizationForm.nfkd, 2)
    assert(contracted.value == "fi")
    contracted.release()
    var rejected = false
    let refused = unicode.normalize("fi", unicode.NormalizationForm.nfkd, 1) catch failure:
        assert(failure.code == strings.output_too_large)
        rejected = true
        recover interop.Owned[str]("")
    refused.release()
    assert(rejected)
    var malformed: u8[3] = [240, 144, 128]
    var empty: u8[]
    var refusing = memory.FixedBuffer.over(empty)
    for form in 0..<4:
        rejected = false
        let malformed_result = unicode.normalize((str)malformed, (unicode.NormalizationForm)(u8)form) catch failure:
            assert(failure.code == utf8.invalid_sequence)
            rejected = true
            recover interop.Owned[str]("")
        malformed_result.release()
        assert(rejected)
        rejected = false
        with refusing:
            let exhausted = unicode.normalize("A", (unicode.NormalizationForm)(u8)form) catch failure:
                assert(failure.code == memory.exhausted)
                rejected = true
                recover interop.Owned[str]("")
            exhausted.release()
        assert(rejected)
    print("ok Unicode normalization conformance and combining-mark stress")
    return 0