dymokomi/luce-compress
Source of release 0.9.0
## Owning Luce facade; all codec implementation remains Luce Base.
import interop
import memory
import binary
import decoder
import encoder
import flate
import segments
import brotli
import gzip
pub let invalid = binary.invalid
pub let corrupt = binary.corrupt
pub let limit_exceeded = binary.limit
pub let unsupported = binary.unsupported
pub let closed = binary.closed
pub let need_input = decoder.need_input
pub let need_output = decoder.need_output
pub let finished = decoder.finished
pub let yielded = decoder.yielded
pub let default_work_limit = decoder.default_work_limit
pub let maximum_work_limit = decoder.maximum_work_limit
pub type Statistics = decoder.Statistics
pub struct Data:
var buffer: binary.Buffer
var input_bytes: i64
var progress: i64 = decoder.finished
var work: i64
var disposed: bool
pub func bytes() -> const u8[]!:
if self.disposed: error(closed, "compression result is closed")
return self.buffer.view()
pub func consumed() -> i64!:
if self.disposed: error(closed, "compression result is closed")
return self.input_bytes
pub func status() -> i64!:
if self.disposed: error(closed, "compression result is closed")
return self.progress
pub func work_units() -> i64!:
if self.disposed: error(closed, "compression result is closed")
return self.work
pub func close():
self.buffer.close()
self.disposed = true
pub let data_type: interop.Type[Data] = interop.Type[Data]("CompressionData", Data.close, closeable = true)
func owned(buffer: binary.Buffer, consumed: i64) -> interop.Reference[Data]!:
let native = new Data(buffer = buffer, input_bytes = consumed) in memory.heap catch failure:
var cleanup = buffer
cleanup.close()
error(failure.code, failure.message)
return try interop.Reference[Data].adopt(native, data_type, memory.heap)
## zlib compression levels: 0 stores, 1 is fastest, 9 smallest (flate's
## libdeflate-style matchers); 6, zlib's default, balances them.
pub let default_level: i64 = 6
## `data` as a zlib stream at `level`, at most `max_output` bytes.
pub func encode(data: const u8[], max_output: i64 = 67108864, level: i64 = default_level) -> interop.Reference[Data]!:
let most = try binary.checked_limit(max_output)
if level < 0 or level > 9: error(invalid, "compression level must be 0 through 9")
let packed = try compress(data, level)
defer free(packed) in memory.heap
if packed.length > most: error(limit_exceeded, "compressed or expanded byte limit exceeded")
var output = binary.Buffer(maximum = most)
output.put(packed) catch failure:
output.close()
error(failure.code, failure.message)
return try owned(output, (i64)data.length)
## The zlib stream at the start of `data` inflated, at most `max_output`
## bytes; bytes after it fail unless `allow_trailing`.
pub func decode(data: const u8[], max_output: i64 = 67108864, allow_trailing: bool = false) -> interop.Reference[Data]!:
var output = binary.Buffer(maximum = try binary.checked_limit(max_output))
if data.length > binary.maximum:
error(limit_exceeded, "compressed input exceeds 1 GiB")
# A stream split into 1 MiB segments (as `compress` writes a large
# buffer) inflates on several threads; any other on one.
if not allow_trailing and data.length > 65536:
let flushes = segments.markers(data)
let room = (flushes + 1) * segments.segment_bytes
if flushes > 0 and room <= output.maximum + segments.segment_bytes:
let within = room if room < output.maximum else output.maximum
output.reserve(within) catch failure:
output.close()
error(failure.code, failure.message)
let made = segments.inflate(data, output.data[..<within], false) catch failure:
output.close()
error(failure.code, failure.message)
if let total = made:
output.used = total
return try owned(output, (i64)data.length)
# One byte past the limit reaches the buffer, which reports the limit.
let used = flate.inflate_zlib_streamed(data, output.maximum + 1, append_output, (void*)&output) catch failure:
output.close()
if failure.code == memory.exhausted or failure.code == limit_exceeded: error(failure.code, failure.message)
error(corrupt, failure.message)
if not allow_trailing and used != data.length:
output.close()
error(corrupt, "trailing bytes after zlib stream")
return try owned(output, (i64)used)
func append_output(context: void*?, bytes: const u8[]) -> !:
let output = (binary.Buffer*)(context else return)
try output.put(bytes)
## Inflate a whole zlib stream into `output`, which must be exactly the
## uncompressed size: what image codecs know from their headers. Base only; the
## table-driven decoder of `flate`. A damaged stream fails `corrupt`; running
## out of memory fails `memory.exhausted`.
pub func inflate(data: const u8[], output: u8[]) -> !:
# A stream split into 1 MiB segments inflates on several threads.
let parallel = segments.inflate(data, output, true) catch failure:
if failure.code == memory.exhausted: error(failure.code, failure.message)
recover none
if parallel != none: return
let used = flate.inflate_zlib(data, output) catch failure:
if failure.code == memory.exhausted: error(failure.code, failure.message)
error(corrupt, failure.message)
if used != data.length:
error(corrupt, "the zlib stream does not match its declared size")
## Deflate `data` at `level` (0 through 9, zlib's) into a fresh zlib stream
## on the heap; the caller frees it. A buffer over 2 MiB is deflated in
## independent 1 MiB segments on several threads (one valid stream, which
## `inflate` and `decode` read back on several threads too). Base only.
pub func compress(data: const u8[], level: i64 = default_level) -> u8[]!:
if level < 0 or level > 9: error(invalid, "compression level must be 0 through 9")
if data.length > binary.maximum: error(limit_exceeded, "input exceeds 1 GiB")
# Over 2 MiB, in 1 MiB segments on several threads (segments.lucb).
if data.length > segments.parallel_above: return try segments.deflate(data, (i32)level)
let scratch = try new u8[flate.deflate_bound(data.length) + 6] --- in memory.heap
defer free(scratch) in memory.heap
let used = try flate.deflate_zlib(data, (i32)level, scratch)
let out = try new u8[used] --- in memory.heap
memory.copy(out, scratch[..<used], used)
return out
## The Brotli stream `data` (RFC 7932) decoded, at most `max_output` bytes; bytes after
## the stream fail.
pub func decode_brotli(data: const u8[], max_output: i64 = 67108864) -> interop.Reference[Data]!:
let plain = try brotli.decode(data, try binary.checked_limit(max_output))
return try adopt_heap(plain, data.length)
## The gzip file `data` (RFC 1952) decoded: every member's data, at most `max_output`
## bytes; bytes after the last member other than zero padding fail.
pub func decode_gzip(data: const u8[], max_output: i64 = 67108864) -> interop.Reference[Data]!:
let plain = try gzip.decode(data, try binary.checked_limit(max_output))
return try adopt_heap(plain, data.length)
## `data` as one gzip member deflated at `level` (0-9).
pub func encode_gzip(data: const u8[], level: i64 = default_level) -> interop.Reference[Data]!:
if level < 0 or level > 9: error(invalid, "compression level must be 0 through 9")
let packed = try gzip.encode(data, (i32)level)
return try adopt_heap(packed, data.length)
## An owned result of heap bytes, which it takes over.
func adopt_heap(bytes: u8[], consumed: usize) -> interop.Reference[Data]!:
return try owned(binary.Buffer(data = bytes, used = bytes.length, maximum = bytes.length), (i64)consumed)
## Checksums are corruption checks, not cryptographic integrity/authentication.
pub func adler32(data: const u8[]) -> i64:
return (i64)flate.adler32(data)
pub func crc32(data: const u8[]) -> i64:
return (i64)flate.crc32(data)
## Chunk-owning facade. Native zero-copy callers use compress_native.Decoder.
pub struct Inflater:
var state: decoder.Decoder*?
pub func init(framing: str = "zlib", max_input: i64 = 1073741824, max_output: i64 = 1073741824) -> !:
let initialized = try decoder.make_decoder(framing, max_input, max_output)
let state = try new decoder.Decoder() in memory.heap
*state = initialized
self.state = state
pub func feed(input: const u8[], capacity: i64 = 65536, final_input: bool = false,
work_limit: i64 = decoder.default_work_limit) -> interop.Reference[Data]!:
let state = self.state else error(closed, "inflater is closed")
try decoder.check_work_limit(work_limit)
if capacity < 0 or capacity > 1048576 or input.length > 1048576:
error(invalid, "feed input/output chunks must be at most 1 MiB")
var buffer = binary.Buffer(maximum = (usize)capacity)
buffer.reserve((usize)capacity) catch failure:
buffer.close()
error(failure.code, failure.message)
## Allocate the result carrier before advancing state: allocation failure
## cannot lose successfully decoded output or create an ambiguous retry.
let result = try owned(buffer, 0)
errdefer result.release()
let chunk = result.value()
let step = try state.step(input, chunk.buffer.data, final_input, work_limit)
chunk.buffer.used = (usize)step.produced
chunk.input_bytes = step.consumed
chunk.progress = step.status
chunk.work = step.work_units
return result
pub func reset() -> !:
let state = self.state else error(closed, "inflater is closed")
try state.reset()
pub func statistics() -> Statistics!:
let state = self.state else error(closed, "inflater is closed")
return state.statistics()
pub func close():
if let state = self.state: free(state) in memory.heap
self.state = none
pub let inflater_type: interop.Type[Inflater] = interop.Type[Inflater]("Inflater", Inflater.close, closeable = true)
pub struct Deflater:
var state: encoder.Encoder
pub func init(framing: str = "zlib", max_input: i64 = 1073741824, max_output: i64 = 1073741824) -> !:
self.state = try encoder.make_encoder(framing, max_input, max_output)
pub func feed(input: const u8[], capacity: i64 = 65536, final_input: bool = false,
work_limit: i64 = decoder.default_work_limit) -> interop.Reference[Data]!:
_ = try self.state.statistics() ## reject closed before allocating
try decoder.check_work_limit(work_limit)
if capacity < 0 or capacity > 1048576 or input.length > 1048576:
error(invalid, "feed input/output chunks must be at most 1 MiB")
var buffer = binary.Buffer(maximum = (usize)capacity)
buffer.reserve((usize)capacity) catch failure:
buffer.close()
error(failure.code, failure.message)
let result = try owned(buffer, 0)
errdefer result.release()
let chunk = result.value()
let step = try self.state.step(input, chunk.buffer.data, final_input, work_limit)
chunk.buffer.used = (usize)step.produced
chunk.input_bytes = step.consumed
chunk.progress = step.status
chunk.work = step.work_units
return result
pub func reset() -> !:
try self.state.reset()
pub func statistics() -> Statistics!:
return try self.state.statistics()
pub func close():
self.state.close()
pub let deflater_type: interop.Type[Deflater] = interop.Type[Deflater]("Deflater", Deflater.close, closeable = true)
test "whole-buffer inflate and compress round trip and reject a wrong size":
let raw = b"the quick brown fox jumps over the lazy dog, again and again and again"
let packed = try compress(raw)
defer free(packed) in memory.heap
let out = try new u8[raw.length] --- in memory.heap
defer free(out) in memory.heap
try inflate(packed, out)
assert((str)out == (str)raw)
var short: u8[10]
var rejected = false
inflate(packed, short) catch failure:
assert(failure.code == corrupt)
rejected = true
assert(rejected)