Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-compress

Source of release 0.9.0

0.9.0 / src / compress.lucb

## Owning Luce facade; all codec implementation remains Luce Base.
import interop
import memory
import binary
import decoder
import encoder
import flate
import segments
import brotli
import gzip

pub let invalid = binary.invalid
pub let corrupt = binary.corrupt
pub let limit_exceeded = binary.limit
pub let unsupported = binary.unsupported
pub let closed = binary.closed
pub let need_input = decoder.need_input
pub let need_output = decoder.need_output
pub let finished = decoder.finished
pub let yielded = decoder.yielded
pub let default_work_limit = decoder.default_work_limit
pub let maximum_work_limit = decoder.maximum_work_limit
pub type Statistics = decoder.Statistics

pub struct Data:
    var buffer: binary.Buffer
    var input_bytes: i64
    var progress: i64 = decoder.finished
    var work: i64
    var disposed: bool

    pub func bytes() -> const u8[]!:
        if self.disposed: error(closed, "compression result is closed")
        return self.buffer.view()

    pub func consumed() -> i64!:
        if self.disposed: error(closed, "compression result is closed")
        return self.input_bytes

    pub func status() -> i64!:
        if self.disposed: error(closed, "compression result is closed")
        return self.progress

    pub func work_units() -> i64!:
        if self.disposed: error(closed, "compression result is closed")
        return self.work

    pub func close():
        self.buffer.close()
        self.disposed = true

pub let data_type: interop.Type[Data] = interop.Type[Data]("CompressionData", Data.close, closeable = true)

func owned(buffer: binary.Buffer, consumed: i64) -> interop.Reference[Data]!:
    let native = new Data(buffer = buffer, input_bytes = consumed) in memory.heap catch failure:
        var cleanup = buffer
        cleanup.close()
        error(failure.code, failure.message)
    return try interop.Reference[Data].adopt(native, data_type, memory.heap)

## zlib compression levels: 0 stores, 1 is fastest, 9 smallest (flate's
## libdeflate-style matchers); 6, zlib's default, balances them.
pub let default_level: i64 = 6

## `data` as a zlib stream at `level`, at most `max_output` bytes.
pub func encode(data: const u8[], max_output: i64 = 67108864, level: i64 = default_level) -> interop.Reference[Data]!:
    let most = try binary.checked_limit(max_output)
    if level < 0 or level > 9: error(invalid, "compression level must be 0 through 9")
    let packed = try compress(data, level)
    defer free(packed) in memory.heap
    if packed.length > most: error(limit_exceeded, "compressed or expanded byte limit exceeded")
    var output = binary.Buffer(maximum = most)
    output.put(packed) catch failure:
        output.close()
        error(failure.code, failure.message)
    return try owned(output, (i64)data.length)

## The zlib stream at the start of `data` inflated, at most `max_output`
## bytes; bytes after it fail unless `allow_trailing`.
pub func decode(data: const u8[], max_output: i64 = 67108864, allow_trailing: bool = false) -> interop.Reference[Data]!:
    var output = binary.Buffer(maximum = try binary.checked_limit(max_output))
    if data.length > binary.maximum:
        error(limit_exceeded, "compressed input exceeds 1 GiB")
    # A stream split into 1 MiB segments (as `compress` writes a large
    # buffer) inflates on several threads; any other on one.
    if not allow_trailing and data.length > 65536:
        let flushes = segments.markers(data)
        let room = (flushes + 1) * segments.segment_bytes
        if flushes > 0 and room <= output.maximum + segments.segment_bytes:
            let within = room if room < output.maximum else output.maximum
            output.reserve(within) catch failure:
                output.close()
                error(failure.code, failure.message)
            let made = segments.inflate(data, output.data[..<within], false) catch failure:
                output.close()
                error(failure.code, failure.message)
            if let total = made:
                output.used = total
                return try owned(output, (i64)data.length)
    # One byte past the limit reaches the buffer, which reports the limit.
    let used = flate.inflate_zlib_streamed(data, output.maximum + 1, append_output, (void*)&output) catch failure:
        output.close()
        if failure.code == memory.exhausted or failure.code == limit_exceeded: error(failure.code, failure.message)
        error(corrupt, failure.message)
    if not allow_trailing and used != data.length:
        output.close()
        error(corrupt, "trailing bytes after zlib stream")
    return try owned(output, (i64)used)

func append_output(context: void*?, bytes: const u8[]) -> !:
    let output = (binary.Buffer*)(context else return)
    try output.put(bytes)

## Inflate a whole zlib stream into `output`, which must be exactly the
## uncompressed size: what image codecs know from their headers. Base only; the
## table-driven decoder of `flate`. A damaged stream fails `corrupt`; running
## out of memory fails `memory.exhausted`.
pub func inflate(data: const u8[], output: u8[]) -> !:
    # A stream split into 1 MiB segments inflates on several threads.
    let parallel = segments.inflate(data, output, true) catch failure:
        if failure.code == memory.exhausted: error(failure.code, failure.message)
        recover none
    if parallel != none: return
    let used = flate.inflate_zlib(data, output) catch failure:
        if failure.code == memory.exhausted: error(failure.code, failure.message)
        error(corrupt, failure.message)
    if used != data.length:
        error(corrupt, "the zlib stream does not match its declared size")

## Deflate `data` at `level` (0 through 9, zlib's) into a fresh zlib stream
## on the heap; the caller frees it. A buffer over 2 MiB is deflated in
## independent 1 MiB segments on several threads (one valid stream, which
## `inflate` and `decode` read back on several threads too). Base only.
pub func compress(data: const u8[], level: i64 = default_level) -> u8[]!:
    if level < 0 or level > 9: error(invalid, "compression level must be 0 through 9")
    if data.length > binary.maximum: error(limit_exceeded, "input exceeds 1 GiB")
    # Over 2 MiB, in 1 MiB segments on several threads (segments.lucb).
    if data.length > segments.parallel_above: return try segments.deflate(data, (i32)level)
    let scratch = try new u8[flate.deflate_bound(data.length) + 6] --- in memory.heap
    defer free(scratch) in memory.heap
    let used = try flate.deflate_zlib(data, (i32)level, scratch)
    let out = try new u8[used] --- in memory.heap
    memory.copy(out, scratch[..<used], used)
    return out

## The Brotli stream `data` (RFC 7932) decoded, at most `max_output` bytes; bytes after
## the stream fail.
pub func decode_brotli(data: const u8[], max_output: i64 = 67108864) -> interop.Reference[Data]!:
    let plain = try brotli.decode(data, try binary.checked_limit(max_output))
    return try adopt_heap(plain, data.length)

## The gzip file `data` (RFC 1952) decoded: every member's data, at most `max_output`
## bytes; bytes after the last member other than zero padding fail.
pub func decode_gzip(data: const u8[], max_output: i64 = 67108864) -> interop.Reference[Data]!:
    let plain = try gzip.decode(data, try binary.checked_limit(max_output))
    return try adopt_heap(plain, data.length)

## `data` as one gzip member deflated at `level` (0-9).
pub func encode_gzip(data: const u8[], level: i64 = default_level) -> interop.Reference[Data]!:
    if level < 0 or level > 9: error(invalid, "compression level must be 0 through 9")
    let packed = try gzip.encode(data, (i32)level)
    return try adopt_heap(packed, data.length)

## An owned result of heap bytes, which it takes over.
func adopt_heap(bytes: u8[], consumed: usize) -> interop.Reference[Data]!:
    return try owned(binary.Buffer(data = bytes, used = bytes.length, maximum = bytes.length), (i64)consumed)

## Checksums are corruption checks, not cryptographic integrity/authentication.
pub func adler32(data: const u8[]) -> i64:
    return (i64)flate.adler32(data)

pub func crc32(data: const u8[]) -> i64:
    return (i64)flate.crc32(data)

## Chunk-owning facade. Native zero-copy callers use compress_native.Decoder.
pub struct Inflater:
    var state: decoder.Decoder*?

    pub func init(framing: str = "zlib", max_input: i64 = 1073741824, max_output: i64 = 1073741824) -> !:
        let initialized = try decoder.make_decoder(framing, max_input, max_output)
        let state = try new decoder.Decoder() in memory.heap
        *state = initialized
        self.state = state

    pub func feed(input: const u8[], capacity: i64 = 65536, final_input: bool = false,
        work_limit: i64 = decoder.default_work_limit) -> interop.Reference[Data]!:
        let state = self.state else error(closed, "inflater is closed")
        try decoder.check_work_limit(work_limit)
        if capacity < 0 or capacity > 1048576 or input.length > 1048576:
            error(invalid, "feed input/output chunks must be at most 1 MiB")
        var buffer = binary.Buffer(maximum = (usize)capacity)
        buffer.reserve((usize)capacity) catch failure:
            buffer.close()
            error(failure.code, failure.message)
        ## Allocate the result carrier before advancing state: allocation failure
        ## cannot lose successfully decoded output or create an ambiguous retry.
        let result = try owned(buffer, 0)
        errdefer result.release()
        let chunk = result.value()
        let step = try state.step(input, chunk.buffer.data, final_input, work_limit)
        chunk.buffer.used = (usize)step.produced
        chunk.input_bytes = step.consumed
        chunk.progress = step.status
        chunk.work = step.work_units
        return result

    pub func reset() -> !:
        let state = self.state else error(closed, "inflater is closed")
        try state.reset()

    pub func statistics() -> Statistics!:
        let state = self.state else error(closed, "inflater is closed")
        return state.statistics()

    pub func close():
        if let state = self.state: free(state) in memory.heap
        self.state = none

pub let inflater_type: interop.Type[Inflater] = interop.Type[Inflater]("Inflater", Inflater.close, closeable = true)

pub struct Deflater:
    var state: encoder.Encoder

    pub func init(framing: str = "zlib", max_input: i64 = 1073741824, max_output: i64 = 1073741824) -> !:
        self.state = try encoder.make_encoder(framing, max_input, max_output)

    pub func feed(input: const u8[], capacity: i64 = 65536, final_input: bool = false,
        work_limit: i64 = decoder.default_work_limit) -> interop.Reference[Data]!:
        _ = try self.state.statistics()  ## reject closed before allocating
        try decoder.check_work_limit(work_limit)
        if capacity < 0 or capacity > 1048576 or input.length > 1048576:
            error(invalid, "feed input/output chunks must be at most 1 MiB")
        var buffer = binary.Buffer(maximum = (usize)capacity)
        buffer.reserve((usize)capacity) catch failure:
            buffer.close()
            error(failure.code, failure.message)
        let result = try owned(buffer, 0)
        errdefer result.release()
        let chunk = result.value()
        let step = try self.state.step(input, chunk.buffer.data, final_input, work_limit)
        chunk.buffer.used = (usize)step.produced
        chunk.input_bytes = step.consumed
        chunk.progress = step.status
        chunk.work = step.work_units
        return result

    pub func reset() -> !:
        try self.state.reset()

    pub func statistics() -> Statistics!:
        return try self.state.statistics()

    pub func close():
        self.state.close()

pub let deflater_type: interop.Type[Deflater] = interop.Type[Deflater]("Deflater", Deflater.close, closeable = true)

test "whole-buffer inflate and compress round trip and reject a wrong size":
    let raw = b"the quick brown fox jumps over the lazy dog, again and again and again"
    let packed = try compress(raw)
    defer free(packed) in memory.heap
    let out = try new u8[raw.length] --- in memory.heap
    defer free(out) in memory.heap
    try inflate(packed, out)
    assert((str)out == (str)raw)
    var short: u8[10]
    var rejected = false
    inflate(packed, short) catch failure:
        assert(failure.code == corrupt)
        rejected = true
    assert(rejected)