Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-compress

Source of release 0.9.1

0.9.1 / tools / codec_tool.lucb

#==============================================================================================
#
#   codec_tool - Brotli, gzip and Zstandard files for the conformance, fuzz and speed runners
#
#   DESCRIPTION:
#       codec_tool batch LIST [LIMIT]
#           Each line of LIST is `MODE INPUT [OUTPUT]`; for each, one line on standard
#           output: `OK <consumed> <produced> <crc32>` or `ERR <code> <message>`. MODE is
#           brotli, gzip or zstd (whole buffer), brotli/IN/OUT or gzip/IN/OUT (incremental,
#           IN input bytes and OUT output bytes per step), or zstd-encode/LEVEL (a zstd
#           frame of the input). OUTPUT, when given, receives the result's bytes. Output
#           is limited to LIMIT bytes (256 MiB by default).
#       codec_tool time FORMAT INPUT ROUNDS
#           The best of ROUNDS whole-buffer runs of INPUT in FORMAT (brotli, gzip, zstd or
#           zstd-encode/LEVEL): bytes out and microseconds.
#
#==============================================================================================
from luce_std import files
import c
import memory
import strings
import time
import binary
import brotli
import flate
import gzip
import zstd

## The default output limit.
let default_most: usize = 268435456

## A decode's result: the input it consumed and its output on the heap.
struct Decoded:
    var consumed: usize
    var output: u8[]

func error_name(code: ErrorCode) -> str:
    if code == binary.corrupt:
        return "corrupt"
    if code == binary.limit:
        return "limit"
    if code == binary.unsupported:
        return "unsupported"
    if code == binary.invalid:
        return "invalid"
    if code == memory.exhausted:
        return "memory"
    return "other"

## Decode `data` as `mode` says, at most `most` bytes.
func run(mode: str, data: const u8[], most: usize) -> Decoded!:
    let parts = try strings.split(mode, '/')
    defer free(parts) in memory.heap
    if parts.length == 1:
        if parts[0] == "brotli":
            let output = try brotli.decode(data, most)
            return Decoded(consumed = data.length, output = output)
        if parts[0] == "gzip":
            let output = try gzip.decode(data, most)
            return Decoded(consumed = data.length, output = output)
        if parts[0] == "zstd":
            let output = try zstd.decode(data, most)
            return Decoded(consumed = data.length, output = output)
        error(binary.invalid, "unknown mode")
    if parts.length == 2 and parts[0] == "zstd-encode":
        let level = (i32)(strings.parse_u64(parts[1]) else 3)
        let output = try zstd.encode(data, level)
        return Decoded(consumed = data.length, output = output)
    let piece = (usize)(strings.parse_u64(parts[1]) else 1)
    let room = (usize)(strings.parse_u64(parts[2]) else 1)
    if parts[0] == "brotli":
        return try stream(data, piece, room, true, most)
    if parts[0] == "gzip":
        return try stream(data, piece, room, false, most)
    error(binary.invalid, "unknown mode")

## Decode `data` through a brotli.Decoder (or a gzip.Decoder), `piece` input bytes and
## `room` output bytes a step at a time.
func stream(data: const u8[], piece: usize, room: usize, use_brotli: bool, most: usize) -> Decoded!:
    var state = brotli.Decoder.open((u64)most)
    defer state.close()
    var unzip = gzip.Decoder.open((u64)most)
    defer unzip.close()
    var output = binary.Buffer(maximum = most)
    errdefer output.close()
    var at: usize = 0
    var offered: usize = 0
    var steps: usize = 0
    while true:
        # Offer up to `piece` new bytes beyond what an earlier step left unconsumed.
        if offered <= at:
            offered = at + piece if at + piece < data.length else data.length
        try output.reserve(room)
        let input = data[at..<offered]
        let space = output.data[output.used..<output.used + room]
        let step = try state.step(input, space, offered == data.length) if use_brotli else try unzip.step(input, space, offered == data.length)
        at += (usize)step.consumed
        output.used += (usize)step.produced
        steps += 1
        if step.status == brotli.finished:
            break
        if step.status == brotli.need_input and at < offered:
            error(binary.invalid, "need_input before consuming the input")
        if steps > 100000000:
            error(binary.invalid, "no progress")
    let result = try new u8[output.used] --- in memory.heap
    memory.copy(result, output.view(), output.used)
    output.close()
    return Decoded(consumed = at, output = result)

func batch(list: c.str, most: usize) -> !:
    let file = try files.read(list)
    defer file.release()
    let lines = try strings.split((str)file.value, '\n')
    defer free(lines) in memory.heap
    for line in lines:
        if line.length == 0:
            continue
        let fields = try strings.split(line, ' ')
        defer free(fields) in memory.heap
        let path = try strings.copy(fields[1])
        defer strings.release(path)
        let input = try files.read((c.str)path)
        defer input.release()
        let decoded = run(fields[0], input.value, most) catch failure:
            print(f"ERR {error_name(failure.code)} {failure.message}")
            continue
        defer free(decoded.output) in memory.heap
        if fields.length > 2:
            let target = try strings.copy(fields[2])
            defer strings.release(target)
            try files.write((c.str)target, decoded.output)
        print(f"OK {decoded.consumed} {decoded.output.length} {flate.crc32(decoded.output)}")

func timing(format: str, path: c.str, rounds: usize) -> !:
    let input = try files.read(path)
    defer input.release()
    var best: u64 = 0
    var size: usize = 0
    for round in 0..<rounds:
        let start = time.now()
        let decoded = try run(format, input.value, default_most)
        let spent = time.since(start)
        size = decoded.output.length
        free(decoded.output) in memory.heap
        best = spent if round == 0 or spent < best else best
    print(f"{size} {best // 1000}")

pub func main(arguments: str[]) -> i32!:
    if (arguments.length == 3 or arguments.length == 4) and arguments[1] == "batch":
        let most = (usize)(strings.parse_u64(arguments[3]) else 0) if arguments.length == 4 else default_most
        try batch((c.str)arguments[2], most)
        return 0
    if arguments.length == 5 and arguments[1] == "time":
        try timing(arguments[2], (c.str)arguments[3], (usize)(strings.parse_u64(arguments[4]) else 5))
        return 0
    print("usage: codec_tool batch LIST [LIMIT] | codec_tool time FORMAT INPUT ROUNDS")
    return 2