Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-compress

Source of release 0.9.1

0.9.1 / src / zstd / module.lucb

#==============================================================================================
#
#   zstd - Zstandard (RFC 8878): whole-buffer decoding and encoding
#
#   DESCRIPTION:
#       `decode` reads every frame of a buffer, skippable frames included, into one
#       heap buffer; `decode_into` reads them into a caller's buffer of known size.
#       The whole format is read: raw, RLE and compressed blocks, raw, RLE, Huffman and
#       treeless literals in one or four streams, FSE sequence tables in predefined,
#       RLE, compressed and repeat modes, repeat offsets and the XXH64 content checksum.
#       Frames that need a dictionary are refused as unsupported. Damaged input fails
#       with `corrupt` and output past a limit with `limit`, never a trap.
#
#       `encode` writes one frame with its content size and checksum: a hash matcher
#       chosen by level (greedy, double-hash or lazy over hash chains) finds the
#       sequences of each 128 KiB block, Huffman codes its literals and FSE codes its
#       sequences in whichever table mode is smallest, and a block that does not shrink
#       is stored raw.
#
#       Fragments, in ORDER:
#
#       module.lucb       the API and the output buffer
#       xxhash.lucb       XXH64, the content checksum
#       bits.lucb         the backward bit reader and the forward bit writer
#       fse.lucb          FSE distributions, decoding and encoding tables
#       huffman.lucb      Huffman tables and literal streams, decoding
#       literals.lucb     the literals section
#       sequences.lucb    the sequences section and sequence execution
#       frame.lucb        frame headers, blocks and frames
#       codes.lucb        the format's length and offset codes
#       match.lucb        the encoder's greedy and double-hash match finders
#       chain.lucb        the lazy match finder over hash chains
#       entropy.lucb      Huffman code lengths and FSE distributions for the encoder
#       block.lucb        the compressor and its blocks
#       section.lucb      writing the literals and sequences sections
#       encoder.lucb      writing a frame
#       tests.lucb        the module's tests
#       vectors.lucb      reference and crafted frames
#
#==============================================================================================
import memory
import binary
from binary import invalid, corrupt, limit, unsupported, closed

## The decoders' default output limit: 64 MiB.
pub let default_limit: usize = 67108864

## Bytes kept writable past a growable output's end, so copies may overrun by a word.
let slack: usize = 32

# mark: The API ===============================================================================

## Every frame of `data` decoded into a heap buffer the caller frees (`free(x) in
## memory.heap`); at most `max_output` bytes, or the call fails with `limit`. Bytes after
## the last frame that do not begin another fail unless `allow_trailing`.
pub func decode(data: const u8[], max_output: usize = default_limit, allow_trailing: bool = false) -> u8[]!:
    var output = Output.growable(max_output)
    errdefer output.close()
    # Frames that declare their sizes are decoded into one allocation.
    if let total = content_size(data):
        if total <= (u64)max_output: try output.reserve((usize)total)
    try decode_frames(data, &output, allow_trailing)
    let result = try new u8[output.used] --- in memory.heap
    if output.used > 0: memory.copy(result, output.data[..<output.used], output.used)
    output.close()
    return result

## Every frame of `data` decoded into `output`; the bytes produced. More output than it
## holds fails with `limit`, bytes after the last frame with `corrupt`.
pub func decode_into(data: const u8[], output: u8[]) -> usize!:
    var target = Output.fixed(output)
    try decode_frames(data, &target, false)
    return target.used

## `data` as one Zstandard frame at `level` (1 fastest .. 9 smallest; 3 by default),
## with its content size and checksum, in a heap buffer the caller frees.
pub func encode(data: const u8[], level: i32 = 3) -> u8[]!:
    if level < 1 or level > 22: error(invalid, "zstd level must be 1 through 22")
    if data.length > binary.maximum: error(limit, "input exceeds 1 GiB")
    return try encode_frame(data, level)

# mark: Output ================================================================================

## Where decoded bytes go: a caller's fixed buffer, or a heap buffer grown up to a limit.
## `data` may extend past `limit` by `slack` bytes for overrunning copies.
struct Output:
    var data: u8[]
    var used: usize
    var limit: usize
    var grows: bool

    ## A heap output of at most `most` bytes.
    static func growable(most: usize) -> Output:
        var made: Output
        made.limit = most
        made.grows = true
        return made

    ## The caller's buffer.
    static func fixed(buffer: u8[]) -> Output:
        var made: Output
        made.data = buffer
        made.limit = buffer.length
        return made

    ## Make room for `count` more bytes plus the slack, when it can grow: fails with
    ## `limit` past the limit. A fixed output is checked by its writers instead.
    func reserve(count: usize) -> !:
        if not self.grows: return
        let wanted = self.used + (count if count < self.limit - self.used else self.limit - self.used)
        if wanted + slack <= self.data.length: return
        var size = self.data.length * 2
        if size < wanted + slack: size = wanted + slack
        if size > self.limit + slack: size = self.limit + slack
        let grown = try new u8[size] --- in memory.heap
        if self.used > 0: memory.copy(grown, self.data[..<self.used], self.used)
        if self.data.length > 0: free(self.data) in memory.heap
        self.data = grown

    func close():
        if self.grows and self.data.length > 0: free(self.data) in memory.heap
        var empty: u8[]
        self.data = empty