dymokomi/luce-compress
Source of release 0.9.0
0.9.0 / tools / codec_tool.lucb
#==============================================================================================
#
# codec_tool - Decode Brotli and gzip files for the conformance, fuzz and speed runners
#
# DESCRIPTION:
# codec_tool batch LIST [LIMIT]
# Each line of LIST is `MODE INPUT [OUTPUT]`; for each, one line on standard
# output: `OK <consumed> <produced> <crc32>` or `ERR <code> <message>`. MODE is
# brotli or gzip (whole buffer), or brotli/IN/OUT or gzip/IN/OUT (incremental,
# IN input bytes and OUT output bytes per step). OUTPUT, when given, receives
# the decoded bytes. Output is limited to LIMIT bytes (256 MiB by default).
# codec_tool time FORMAT INPUT ROUNDS
# The best of ROUNDS whole-buffer decodes of INPUT (brotli or gzip): bytes out and
# microseconds.
#
#==============================================================================================
from luce_std import files
import c
import memory
import strings
import time
import binary
import brotli
import flate
import gzip
## The default output limit.
let default_most: usize = 268435456
## A decode's result: the input it consumed and its output on the heap.
struct Decoded:
var consumed: usize
var output: u8[]
func error_name(code: ErrorCode) -> str:
if code == binary.corrupt:
return "corrupt"
if code == binary.limit:
return "limit"
if code == binary.unsupported:
return "unsupported"
if code == binary.invalid:
return "invalid"
if code == memory.exhausted:
return "memory"
return "other"
## Decode `data` as `mode` says, at most `most` bytes.
func run(mode: str, data: const u8[], most: usize) -> Decoded!:
let parts = try strings.split(mode, '/')
defer free(parts) in memory.heap
if parts.length == 1:
if parts[0] == "brotli":
let output = try brotli.decode(data, most)
return Decoded(consumed = data.length, output = output)
if parts[0] == "gzip":
let output = try gzip.decode(data, most)
return Decoded(consumed = data.length, output = output)
error(binary.invalid, "unknown mode")
let piece = (usize)(strings.parse_u64(parts[1]) else 1)
let room = (usize)(strings.parse_u64(parts[2]) else 1)
if parts[0] == "brotli":
return try stream(data, piece, room, true, most)
if parts[0] == "gzip":
return try stream(data, piece, room, false, most)
error(binary.invalid, "unknown mode")
## Decode `data` through a brotli.Decoder (or a gzip.Decoder), `piece` input bytes and
## `room` output bytes a step at a time.
func stream(data: const u8[], piece: usize, room: usize, use_brotli: bool, most: usize) -> Decoded!:
var state = brotli.Decoder.open((u64)most)
defer state.close()
var unzip = gzip.Decoder.open((u64)most)
defer unzip.close()
var output = binary.Buffer(maximum = most)
errdefer output.close()
var at: usize = 0
var offered: usize = 0
var steps: usize = 0
while true:
# Offer up to `piece` new bytes beyond what an earlier step left unconsumed.
if offered <= at:
offered = at + piece if at + piece < data.length else data.length
try output.reserve(room)
let input = data[at..<offered]
let space = output.data[output.used..<output.used + room]
let step = try state.step(input, space, offered == data.length) if use_brotli else try unzip.step(input, space, offered == data.length)
at += (usize)step.consumed
output.used += (usize)step.produced
steps += 1
if step.status == brotli.finished:
break
if step.status == brotli.need_input and at < offered:
error(binary.invalid, "need_input before consuming the input")
if steps > 100000000:
error(binary.invalid, "no progress")
let result = try new u8[output.used] --- in memory.heap
memory.copy(result, output.view(), output.used)
output.close()
return Decoded(consumed = at, output = result)
func batch(list: c.str, most: usize) -> !:
let file = try files.read(list)
defer file.release()
let lines = try strings.split((str)file.value, '\n')
defer free(lines) in memory.heap
for line in lines:
if line.length == 0:
continue
let fields = try strings.split(line, ' ')
defer free(fields) in memory.heap
let path = try strings.copy(fields[1])
defer strings.release(path)
let input = try files.read((c.str)path)
defer input.release()
let decoded = run(fields[0], input.value, most) catch failure:
print(f"ERR {error_name(failure.code)} {failure.message}")
continue
defer free(decoded.output) in memory.heap
if fields.length > 2:
let target = try strings.copy(fields[2])
defer strings.release(target)
try files.write((c.str)target, decoded.output)
print(f"OK {decoded.consumed} {decoded.output.length} {flate.crc32(decoded.output)}")
func timing(format: str, path: c.str, rounds: usize) -> !:
let input = try files.read(path)
defer input.release()
var best: u64 = 0
var size: usize = 0
for round in 0..<rounds:
let start = time.now()
let decoded = try run(format, input.value, default_most)
let spent = time.since(start)
size = decoded.output.length
free(decoded.output) in memory.heap
best = spent if round == 0 or spent < best else best
print(f"{size} {best // 1000}")
pub func main(arguments: str[]) -> i32!:
if (arguments.length == 3 or arguments.length == 4) and arguments[1] == "batch":
let most = (usize)(strings.parse_u64(arguments[3]) else 0) if arguments.length == 4 else default_most
try batch((c.str)arguments[2], most)
return 0
if arguments.length == 5 and arguments[1] == "time":
try timing(arguments[2], (c.str)arguments[3], (usize)(strings.parse_u64(arguments[4]) else 5))
return 0
print("usage: codec_tool batch LIST [LIMIT] | codec_tool time FORMAT INPUT ROUNDS")
return 2