Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-compress

Source of release 0.9.1

0.9.1 / tools / zstd_vectors.py

#!/usr/bin/env python3
"""Generate src/zstd/vectors.lucb: Zstandard test vectors as tests of the module.

  tools/zstd_vectors.py [--zstd ZSTD_CLI] [--donor DIR]

Three kinds of frame:

* The zstd repository's golden decompression files (tests/golden-decompression and
  tests/golden-decompression-errors, from --donor, a checkout of facebook/zstd): small
  frames exercising corner cases; the error files must fail.
* Reference frames: the `zstd` CLI compressing deterministic inputs (text and binary
  made by a small generator the tests repeat in Luce) at a range of levels, without a
  checksum or content size, with a long window, as several frames and with a skippable
  frame between them.
* The output's length and CRC-32 are what each test expects.

Each vector becomes a `test` block (compiled only for tests). The frames are held as
little-endian 64-bit words.
"""
import argparse
import shutil
import subprocess
import zlib
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
OUTPUT = ROOT / "src/zstd/vectors.lucb"


def text(size, seed):
    """Deterministic words: an LCG over a word list (the tests' `sample_text`)."""
    words = [b"the", b"zstd", b"window", b"of", b"and", b"frame", b"<div>", b"</div>", b"sequence", b"\xc3\xa9t\xc3\xa9",
             b"\n", b"for", b"a", b"literal", b"0123", b"http://", b".com/", b"ing ", b"The ", b"block"]
    out, state = bytearray(), seed
    while len(out) < size:
        state = (state * 1103515245 + 12345) & 0x7FFFFFFF
        out += words[(state >> 16) % len(words)] + b" "
    return bytes(out[:size])


def binary(size, seed):
    """Records: a counter, a small varying field and a noisy byte."""
    out, state = bytearray(), seed
    while len(out) < size:
        state = (state * 1103515245 + 12345) & 0x7FFFFFFF
        out += (len(out) // 8).to_bytes(4, "little") + bytes([(state >> 16) & 3, 0, (state >> 20) & 0xFF, 7])
    return bytes(out[:size])


def zstd(cli, data, *flags):
    return subprocess.run([cli, "-q", "-c", *flags], input=data, capture_output=True, check=True).stdout


def words(frame):
    padded = frame + bytes(-len(frame) % 8)
    values = [int.from_bytes(padded[i:i + 8], "little") for i in range(0, len(padded), 8)]
    lines = []
    for start in range(0, len(values), 6):
        lines.append("        " + ", ".join(f"0x{v:016x}" for v in values[start:start + 6]) + ",")
    return "\n".join(lines)


def block(name, frame, output):
    """A test that decodes `frame` to `output`, or expects it to fail when output is None."""
    count = len(words(frame).replace("\n", "").split(",")) - 1
    lines = [f'test "{name}":', f"    let frame: u64[{count}] = [", words(frame), "    ]"]
    view = f"u8[]((u8*)&frame[0], {len(frame)})"
    if output is None:
        lines.append(f"    assert(failure_of({view}) == corrupt)")
    else:
        lines.append(f"    try check_vector({view}, {len(output)}, {zlib.crc32(output)})")
    return "\n".join(lines) + "\n"


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--zstd", default=shutil.which("zstd"))
    parser.add_argument("--donor", default=str(ROOT.parent / ".donors/zstd"))
    args = parser.parse_args()
    version = subprocess.run([args.zstd, "--version"], capture_output=True, text=True, check=True).stdout.split()
    version = next(word for word in version if word.startswith("v")).rstrip(",")
    tests = []
    donor = Path(args.donor) / "tests"
    for path in sorted((donor / "golden-decompression").glob("*.zst")):
        if path.stat().st_size > 4096:
            continue  # block-128k is built by a test of its own
        frame = path.read_bytes()
        output = subprocess.run([args.zstd, "-d", "-c"], input=frame, capture_output=True, check=True).stdout
        tests.append(block(f"golden {path.name}", frame, output))
    for path in sorted((donor / "golden-decompression-errors").glob("*.zst")):
        tests.append(block(f"golden error {path.name}", path.read_bytes(), None))
    sample = text(12000, 7)
    records = binary(12000, 11)
    for level in [1, 3, 7, 12, 19]:
        tests.append(block(f"text, level {level}", zstd(args.zstd, sample, f"-{level}"), sample))
    tests.append(block("binary, level 3", zstd(args.zstd, records, "-3"), records))
    tests.append(block("binary, level 22", zstd(args.zstd, records, "--ultra", "-22"), records))
    tests.append(block("text without checksum", zstd(args.zstd, sample, "--no-check"), sample))
    tests.append(block("text, long window", zstd(args.zstd, sample, "--long=24", "-5"), sample))
    # Streamed: no content size in the header.
    streamed = subprocess.run([args.zstd, "-q", "-c", "--no-content-size", "-3"], input=sample, capture_output=True, check=True).stdout
    tests.append(block("text without content size", streamed, sample))
    # Two frames with a skippable frame between them.
    skippable = (0x184D2A53).to_bytes(4, "little") + (5).to_bytes(4, "little") + b"luce!"
    joined = zstd(args.zstd, sample[:7000], "-3") + skippable + zstd(args.zstd, records[:9000], "-1")
    tests.append(block("two frames and a skippable frame", joined, sample[:7000] + records[:9000]))
    header = f"""#==============================================================================================
#
#   zstd/vectors - Reference and golden frames (generated tests)
#
#   DESCRIPTION:
#       Generated by tools/zstd_vectors.py; do not edit. The zstd repository's golden
#       decompression files, and frames from the zstd CLI ({version}) over deterministic
#       inputs. Each decodes to the output's length and CRC-32, or fails as corrupt (the
#       frame is held as little-endian 64-bit words).
#
#==============================================================================================
"""
    OUTPUT.write_text(header + "\n" + "\n".join(tests))
    print(f"wrote {OUTPUT} ({len(tests)} vectors)")


if __name__ == "__main__":
    main()