Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-compress

Source of release 0.9.0

0.9.0 / tests / stress_driver.lucb

## Test-only generated-stream pipeline. Never retains a whole source or result.
import native as codec
import thread
import time
import strings

var ready: @u64
var start: @u64

struct Measure:
    var histogram: u64[64]
    var maximum: u64
    var nanoseconds: u64
    var calls: u64
    var units: u64
    var yields: u64

func record(measure: Measure*, step: codec.Step, elapsed: u64, allowance: i64):
    assert(step.work_units > 0 and step.work_units <= allowance)
    if step.status == codec.yielded: assert(step.work_units == allowance)
    var bucket: usize
    var rounded = elapsed
    while rounded > 1 and bucket < 63:
        rounded = (rounded + 1) // 2
        bucket += 1
    measure.histogram[bucket] += 1
    if elapsed > measure.maximum: measure.maximum = elapsed
    measure.nanoseconds += elapsed
    measure.calls += 1
    measure.units += (u64)step.work_units
    if step.status == codec.yielded: measure.yields += 1

struct Generator:
    var seed: u32
    var noise: bool

    func next() -> u8:
        if not self.noise: return 65
        self.seed ^= self.seed << 13
        self.seed ^= self.seed >> 17
        self.seed ^= self.seed << 5
        return (u8)self.seed

struct Context:
    var bytes: i64
    var rounds: i64
    var work: i64
    var wrapped: bool
    var noise: bool
    var seed: u32
    var encoded: u64
    var encode: Measure
    var decode: Measure

func pipeline(context: Context*) -> !:
    let framing = "zlib" if context.wrapped else "raw"
    let packed_limit = context.bytes * 2 + 1024
    var encoder = try codec.make_encoder(framing, context.bytes, packed_limit)
    defer encoder.close()
    var decoder = try codec.make_decoder(framing, packed_limit, context.bytes)
    defer decoder.close()
    var input: u8[32768]
    var packed: u8[32769]
    var output: u8[32769]
    _ = ready.add(1, .release)
    while start.load(.acquire) == 0: thread.sleep(1)
    if start.load(.acquire) == 2: return
    for round in 0..<context.rounds:
        if round > 0:
            try encoder.reset()
            try decoder.reset()
            assert((try encoder.statistics()).input_bytes == 0)
            assert(decoder.statistics().input_bytes == 0)
        var source = Generator(seed = context.seed + (u32)round, noise = context.noise)
        var expected = Generator(seed = context.seed + (u32)round, noise = context.noise)
        var generated: i64
        var accepted: i64
        var checked: i64
        var encoded: i64
        var input_at: usize
        var input_size: usize
        var decoded_end = false
        while true:
            if input_at == input_size and generated < context.bytes:
                input_size = (usize)(context.bytes - generated)
                if input_size > input.length: input_size = input.length
                for index in 0..<input_size: input[index] = source.next()
                generated += (i64)input_size
                input_at = 0
            packed[32768] = 173
            let encode_start = time.now()
            let step = try encoder.step(input[input_at..<input_size], packed[..<32768],
                generated == context.bytes, context.work)
            record(&context.encode, step, time.since(encode_start), context.work)
            assert(step.consumed >= 0 and step.consumed <= (i64)(input_size - input_at))
            assert(step.produced >= 0 and step.produced <= 32768 and packed[32768] == 173)
            assert(step.consumed <= 258 * step.work_units and step.produced <= 8 * step.work_units)
            for index in input_at..<input_at + (usize)step.consumed: input[index] = 221
            input_at += (usize)step.consumed
            accepted += step.consumed
            encoded += step.produced
            let encode_stats = try encoder.statistics()
            assert(encode_stats.input_bytes == accepted and encode_stats.output_bytes == encoded)
            if decoded_end: assert(step.produced == 0)
            if not decoded_end and (step.produced > 0 or step.status == codec.finished):
                var packed_at: usize
                while true:
                    output[32768] = 173
                    let decode_start = time.now()
                    let decoded = try decoder.step(packed[packed_at..<(usize)step.produced],
                        output[..<32768], step.status == codec.finished, context.work)
                    record(&context.decode, decoded, time.since(decode_start), context.work)
                    assert(decoded.consumed >= 0 and decoded.consumed <= step.produced - (i64)packed_at)
                    assert(decoded.produced >= 0 and decoded.produced <= 32768 and output[32768] == 173)
                    assert(decoded.consumed <= 8 * decoded.work_units and decoded.produced <= 258 * decoded.work_units)
                    for index in 0..<(usize)decoded.produced:
                        assert(checked < context.bytes and output[index] == expected.next())
                        checked += 1
                        output[index] = 222
                    for index in packed_at..<packed_at + (usize)decoded.consumed: packed[index] = 223
                    packed_at += (usize)decoded.consumed
                    assert(decoder.statistics().output_bytes == checked)
                    if decoded.status == codec.finished:
                        ## Raw EOB can reach the consumer before the producer's
                        ## final zero-output state transitions have completed.
                        assert(packed_at == (usize)step.produced)
                        decoded_end = true
                        break
                    if decoded.status == codec.need_input:
                        assert(packed_at == (usize)step.produced and step.status != codec.finished)
                        break
                    if decoded.status == codec.need_output: assert(decoded.produced == 32768)
                    else: assert(decoded.status == codec.yielded)
            if step.status == codec.finished:
                assert(decoded_end and accepted == context.bytes and checked == context.bytes)
                assert(encode_stats.finished and decoder.statistics().finished)
                assert(decoder.statistics().input_bytes == encoded)
                context.encoded += (u64)encoded
                break
            if step.status == codec.need_input: assert(input_at == input_size and generated < context.bytes)
            elif step.status == codec.need_output: assert(step.produced == 32768)
            else: assert(step.status == codec.yielded)

func worker(raw: void*?):
    let context = (Context*)(raw else trap("missing stress context"))
    pipeline(context) catch failure: trap(failure.message)

func merge(total: Measure*, item: Measure*):
    for bucket in 0..<64: total.histogram[bucket] += item.histogram[bucket]
    if item.maximum > total.maximum: total.maximum = item.maximum
    total.nanoseconds += item.nanoseconds
    total.calls += item.calls
    total.units += item.units
    total.yields += item.yields

func report(name: str, measure: Measure*):
    print(f"CALLS {name} {measure.calls} {measure.units} {measure.yields} {measure.nanoseconds} {measure.maximum}")
    for bucket in 0..<64: print(f"HIST {name} {bucket} {measure.histogram[bucket]}")

pub func main(arguments: str[]) -> i32!:
    assert(arguments.length == 7)
    let count = strings.parse_i64(arguments[1]) else trap("worker count")
    let bytes = strings.parse_i64(arguments[2]) else trap("bytes per stream")
    let rounds = strings.parse_i64(arguments[3]) else trap("rounds")
    let allowance = strings.parse_i64(arguments[6]) else trap("work allowance")
    assert(count >= 1 and count <= 8 and bytes >= 0 and bytes <= 4294967297)
    assert(rounds >= 1 and rounds <= 32 and allowance >= 1 and allowance <= 65536)
    assert(arguments[4] == "raw" or arguments[4] == "zlib")
    assert(arguments[5] == "repeat" or arguments[5] == "noise")
    var contexts: Context[8]
    var handles: thread.Handle[8]
    for index in 0..<count:
        contexts[index] = Context(bytes = bytes, rounds = rounds, work = allowance,
            wrapped = arguments[4] == "zlib", noise = arguments[5] == "noise", seed = (u32)index + 19511950)
        handles[index] = thread.spawn(worker, &contexts[index], stack = 524288) catch failure:
            start.store(2, .release)
            for previous in 0..<index: try handles[previous].join()
            error(failure.code, failure.message)
    while ready.load(.acquire) != (u64)count: thread.sleep(1)
    let began = time.now()
    start.store(1, .release)
    for index in 0..<count: try handles[index].join()
    let elapsed = time.since(began)
    var encoded: u64
    var encode: Measure
    var decode: Measure
    for index in 0..<count:
        encoded += contexts[index].encoded
        merge(&encode, &contexts[index].encode)
        merge(&decode, &contexts[index].decode)
    print(f"RESULT {count} {bytes} {rounds} {count * bytes * rounds} {encoded} {elapsed} {ready.load(.acquire)}")
    report("encode", &encode)
    report("decode", &decode)
    return 0