Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-compress

Source of release 0.9.0

0.9.0 / tests / check_work.py

#!/usr/bin/env python3
"""Independent codec fixtures with bounded dispatch budgets, including zero-I/O yields."""
from pathlib import Path
import random
import subprocess
import tempfile
import zlib
from check_codecs import dynamic_literal


def check(decoder, encoder):
    decoder, encoder = Path(decoder).resolve(), Path(encoder).resolve()
    rng = random.Random(19514096)
    checks = 0
    default_yields = [False, False]
    empty_yields = [False, False]
    with tempfile.TemporaryDirectory(prefix="luce-compress-work-") as temporary:
        root = Path(temporary)
        source, expected, destination = root / "source", root / "expected", root / "encoded"

        def run(command, maximum_steps, kind, work, reject=False):
            nonlocal checks
            result = subprocess.run([str(item) for item in command], capture_output=True, timeout=30)
            checks += 1
            for marker in [b"AddressSanitizer", b"UndefinedBehaviorSanitizer", b"runtime error:"]:
                assert marker not in result.stderr, result.stderr
            if reject:
                assert result.returncode == 1 and result.stdout.startswith(b"REJECT "), (result.stdout, result.stderr)
                return None
            assert result.returncode == 0, (command, result.stdout, result.stderr)
            fields = result.stdout.decode().strip().split()
            assert len(fields) == 7 and fields[0] == "OK", fields
            consumed, produced, steps, yields, empty, units = map(int, fields[1:])
            assert 1 <= steps <= maximum_steps and 0 <= empty <= yields < steps, fields
            assert steps <= units <= steps * (31 if work == 0 else work), fields
            if work == 4096 and yields: default_yields[kind] = True
            if empty: empty_yields[kind] = True
            return consumed, produced

        def decode(packed, plain, framing, work, chunks=(65536, 65536), end="end",
                   reject=False, max_input=None, max_output=None, suffix=b""):
            source.write_bytes(packed + suffix)
            expected.write_bytes(plain)
            limits = (len(packed) if max_input is None else max_input,
                      len(plain) if max_output is None else max_output)
            counts = run([decoder, source, expected, framing, *chunks, *limits, end,
                          "invalid" if reject else "valid", work],
                         16 * (len(packed) + len(plain) + 1) + 128, 0, work, reject)
            if not reject: assert counts == (len(packed), len(plain)), counts

        def encode(plain, framing, work, chunks=(65536, 65536), end="end",
                   reject=False, max_input=None, max_output=2097152):
            source.write_bytes(plain)
            counts = run([encoder, source, destination, framing, *chunks,
                          len(plain) if max_input is None else max_input, max_output, end, work],
                         16 * (len(plain) + 1) + 128, 1, work, reject)
            if reject: return None
            packed = destination.read_bytes()
            assert counts == (len(plain), len(packed)), counts
            oracle = zlib.decompressobj(15 if framing == "zlib" else -15)
            decoded = oracle.decompress(packed, len(plain) + 1)
            assert decoded == plain and oracle.eof and not oracle.unused_data and not oracle.unconsumed_tail
            return packed

        corpus = [b"", b"A", b"abc\0" * 259, rng.randbytes(4097), rng.randbytes(65537),
                  b"a" * 131073, bytes(range(256)) * 257]
        def chunks_for(work):
            # Borrowed buffers are overwritten after every step. Avoid quadratic
            # test-driver copying of a huge suffix when one dispatch consumes a
            # byte; native output still has room for many dispatches. Default/max
            # allowance fixtures retain the full 64 KiB input-span coverage.
            if work == 0: return 0, 0
            return (257, 65536) if work in [1, 2, 7] else (65536, 65536)

        for framing, window in [("raw", -15), ("zlib", 15)]:
            for plain in corpus:
                reference = encode(plain, framing, 65536)
                for work in [1, 2, 7, 0, 4096, 65536]:
                    assert encode(plain, framing, work, chunks_for(work)) == reference
                for level, strategy in [(0, zlib.Z_DEFAULT_STRATEGY), (6, zlib.Z_FIXED),
                                        (6, zlib.Z_DEFAULT_STRATEGY), (6, zlib.Z_HUFFMAN_ONLY)]:
                    writer = zlib.compressobj(level, zlib.DEFLATED, window, 8, strategy)
                    packed = writer.compress(plain) + writer.flush()
                    for work in [1, 2, 7, 0, 4096, 65536]:
                        decode(packed, plain, framing, work, chunks_for(work))

            # Empty stored blocks spend CPU even when they emit no bytes.
            raw = b"\x00\x00\x00\xff\xff" * 4096 + b"\x01\x00\x00\xff\xff"
            packed = raw if framing == "raw" else b"\x78\x01" + raw + b"\x00\x00\x00\x01"
            assert zlib.decompress(packed, window) == b""
            for work in [1, 7, 4096]: decode(packed, b"", framing, work, suffix=b"next object")

            # Every first split and truncated prefix of fixed/dynamic fixtures,
            # with a one-operation budget and both ordinary and delayed EOF.
            for wrapped, plain in [(zlib.compress(b"abcabcabc"), b"abcabcabc"),
                                   (dynamic_literal(b"A"), b"A"), (dynamic_literal(), b"")]:
                packed = wrapped if framing == "zlib" else wrapped[2:-4]
                for cut in range(len(packed) + 1):
                    decode(packed, plain, framing, 1, (-cut - 1, 1))
                for cut in range(len(packed)):
                    decode(packed[:cut], b"", framing, 1, (1, 1), end="late", reject=True, max_output=1024)
                decode(packed, plain, framing, 1, (1, 1), end="late", suffix=b"next stream")
                if plain:
                    decode(packed, b"", framing, 1, reject=True, max_output=len(plain) - 1)
                decode(packed, b"", framing, 1, reject=True, max_input=len(packed) - 1, max_output=1024)

            short = b"final marker across yields"
            reference = encode(short, framing, 65536)
            for cut in range(len(short) + 1):
                assert encode(short, framing, 1, (-cut - 1, 1), end="late") == reference
            encode(short, framing, 1, reject=True, max_input=len(short) - 1)
            encode(short, framing, 1, reject=True, max_output=len(reference) - 1)
            # A reserved block must reject, not yield indefinitely.
            invalid = b"\x07" if framing == "raw" else b"\x78\x01\x07"
            decode(invalid, b"", framing, 1, reject=True, max_output=1024)

        wrapped = zlib.compress(b"partial output remains quarantined" * 100)
        for index in range(len(wrapped) - 4, len(wrapped)):
            changed = bytearray(wrapped); changed[index] ^= 1
            decode(bytes(changed), b"", "zlib", 1, reject=True, max_output=65536)

    assert all(default_yields) and all(empty_yields), (default_yields, empty_yields)
    print(f"PASS {checks} cooperative-work oracle/split/error cases; bounded dispatches, zero-I/O/default yields and exact resumed bytes", flush=True)


if __name__ == "__main__":
    import sys
    check(sys.argv[1], sys.argv[2])