Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-compress

Source of release 0.9.0

0.9.0 / tests / check_large.py

#!/usr/bin/env python3
"""Independent larger file-stream fixtures; bounded Python/native working chunks."""
import argparse
from pathlib import Path
import random
import subprocess
import tempfile
import zlib

MIB = 1024 * 1024


def verify_encoded(original, actual, window, amount):
    reader = zlib.decompressobj(window)
    expanded = 0
    while chunk := actual.read(65536):
        while chunk:
            previous = len(chunk)
            output = reader.decompress(chunk, 65536)
            chunk = reader.unconsumed_tail
            assert output or len(chunk) < previous, "oracle made no progress"
            assert expanded + len(output) <= amount
            assert original.read(len(output)) == output
            expanded += len(output)
            assert not reader.unused_data, "unexpected trailing compressed bytes"
    # Empty unconsumed_tail is not an EOF proof; drain retained output as needed.
    while not reader.eof:
        output = reader.decompress(b"", 65536)
        if not output: break
        assert expanded + len(output) <= amount
        assert original.read(len(output)) == output
        expanded += len(output)
    assert reader.eof and expanded == amount and not original.read(1)


def check(driver, full=False):
    driver = Path(driver).resolve()
    checks = 0
    with tempfile.TemporaryDirectory(prefix="luce-compress-large-") as temporary:
        root = Path(temporary)
        plain, reference, encoded, decoded = [root / name for name in ["plain", "reference", "encoded", "decoded"]]

        def run(operation, source, target, framing, input_limit, output_limit, reject=False):
            nonlocal checks
            result = subprocess.run([str(driver), operation, str(source), str(target), framing,
                                     str(input_limit), str(output_limit)], capture_output=True, timeout=90)
            checks += 1
            for marker in [b"AddressSanitizer", b"UndefinedBehaviorSanitizer", b"runtime error:"]:
                assert marker not in result.stderr, result.stderr
            if reject:
                assert result.returncode == 1 and result.stdout.startswith(b"REJECT "), (result.stdout, result.stderr)
            else:
                assert result.returncode == 0, (result.stdout, result.stderr)
                fields = result.stdout.decode().split()
                assert len(fields) == 3 and fields[0] == "OK", fields
                assert int(fields[1]) == source.stat().st_size and int(fields[2]) == target.stat().st_size

        for amount in ([MIB + 1, 16 * MIB + 3] if full else [MIB + 1]):
            for framing, window in [("raw", -15), ("zlib", 15)]:
                for pattern in ["repeat", "noise"]:
                    rng = random.Random(19510002)
                    writer = zlib.compressobj(6, zlib.DEFLATED, window)
                    with plain.open("wb") as original, reference.open("wb") as packed:
                        remaining = amount
                        while remaining:
                            size = min(65536, remaining)
                            chunk = rng.randbytes(size) if pattern == "noise" else b"A" * size
                            original.write(chunk)
                            packed.write(writer.compress(chunk))
                            remaining -= size
                        packed.write(writer.flush())
                    packed_size = reference.stat().st_size
                    run("decode", reference, decoded, framing, packed_size, amount)
                    with plain.open("rb") as original, decoded.open("rb") as actual:
                        while chunk := original.read(65536):
                            assert actual.read(len(chunk)) == chunk
                        assert not actual.read(1)
                    run("encode", plain, encoded, framing, amount, amount * 2 + 1024)
                    native_size = encoded.stat().st_size
                    with plain.open("rb") as original, encoded.open("rb") as actual:
                        verify_encoded(original, actual, window, amount)
                    # Keep the full-size positive output intact until both oracles
                    # have verified it; all negative output is disposable afterward.
                    run("decode", reference, decoded, framing, packed_size - 1, amount, reject=True)
                    run("decode", reference, decoded, framing, packed_size, amount - 1, reject=True)
                    run("encode", plain, encoded, framing, amount - 1, amount * 2 + 1024, reject=True)
                    run("encode", plain, encoded, framing, amount, native_size - 1, reject=True)
                    print(f"LARGE bytes={amount} framing={framing} pattern={pattern} native_bytes={native_size}", flush=True)
    print(f"PASS {checks} independent larger-stream oracle/budget cases; bounded file I/O and exact bytes", flush=True)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("driver", type=Path)
    parser.add_argument("--full", action="store_true")
    args = parser.parse_args()
    check(args.driver, args.full)