Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-compress

Source of release 0.9.0

0.9.0 / tests / check_stream.py

#!/usr/bin/env python3
"""Independent incremental/raw/zlib fixtures; short-lived chunks and exact budgets."""
from pathlib import Path
import random
import subprocess
import tempfile
import zlib
from check_codecs import Bits, dynamic_literal


def check(executable):
    executable = Path(executable).resolve()
    random_source = random.Random(19501951)
    checks = 0
    with tempfile.TemporaryDirectory(prefix="luce-compress-stream-") as tmp:
        packed_file, plain_file = Path(tmp) / "packed", Path(tmp) / "plain"

        def invoke(packed, plain=b"", framing="zlib", inchunk=0, outchunk=0,
                   max_input=None, max_output=None, end="end", suffix=b"", reject=False):
            nonlocal checks
            packed_file.write_bytes(packed + suffix)
            plain_file.write_bytes(plain)
            if max_input is None: max_input = len(packed)
            if max_output is None: max_output = 1048576 if reject else len(plain)
            result = subprocess.run([str(executable), str(packed_file), str(plain_file), framing,
                                     str(inchunk), str(outchunk), str(max_input), str(max_output),
                                     end, "invalid" if reject else "valid"],
                                    capture_output=True, timeout=30)
            checks += 1
            for marker in [b"AddressSanitizer", b"UndefinedBehaviorSanitizer", b"runtime error:"]:
                assert marker not in result.stderr, result.stderr
            if reject:
                assert result.returncode == 1 and result.stdout.startswith(b"REJECT "), (result.returncode, result.stdout, result.stderr)
            else:
                assert result.returncode == 0, (framing, len(packed), inchunk, outchunk, result.stdout, result.stderr)
                fields = result.stdout.decode().strip().split()
                assert fields[0] == "OK" and int(fields[1]) == len(packed) and int(fields[2]) == len(plain), fields

        corpus = [b"", b"A", b"abracadabra\0" * 31, b"x" * 4097,
                  bytes(range(256)) * 300, random_source.randbytes(4097), random_source.randbytes(65537)]
        for framing, window in [("zlib", 15), ("raw", -15)]:
            for plain in corpus:
                for level, strategy in [(0, zlib.Z_DEFAULT_STRATEGY), (6, zlib.Z_FIXED),
                                        (6, zlib.Z_DEFAULT_STRATEGY), (6, zlib.Z_HUFFMAN_ONLY)]:
                    encoder = zlib.compressobj(level, zlib.DEFLATED, window, 8, strategy)
                    packed = encoder.compress(plain) + encoder.flush()
                    for inchunk, outchunk in [(1, 257), (17, 1), (0, 0), (65536, 65536)]:
                        invoke(packed, plain, framing, inchunk, outchunk)
                    if len(plain) <= 4097:
                        invoke(packed, plain, framing, 1, 1, end="late")
                    invoke(packed, plain, framing, 7, 13, suffix=b"the next object\0\xff")
                    invoke(packed, framing=framing, inchunk=1, outchunk=17,
                           max_input=len(packed) - 1, reject=True)
                    if plain:
                        invoke(packed, framing=framing, inchunk=11, outchunk=1,
                               max_output=len(plain) - 1, reject=True)

            # Every first split, including empty input, and every truncated prefix.
            short = [(zlib.compress(b"one two one two"), b"one two one two"),
                     (zlib.compress(b"abc", 0), b"abc"), (dynamic_literal(b"A"), b"A"),
                     (dynamic_literal(), b"")]
            for wrapped, plain in short:
                packed = wrapped if framing == "zlib" else wrapped[2:-4]
                for cut in range(len(packed) + 1):
                    for outchunk in [0, 1, 7]:
                        invoke(packed, plain, framing, -cut - 1, outchunk)
                for cut in range(len(packed)):
                    for inchunk, outchunk, end in [(1, 1, "end"), (3, 0, "late")]:
                        invoke(packed[:cut], framing=framing, inchunk=inchunk, outchunk=outchunk,
                               end=end, reject=True)

            # Multiple compressed/empty stored blocks with history across flushes.
            encoder = zlib.compressobj(wbits=window)
            base = random_source.randbytes(32768)
            pieces = [base, base, b"a" * 40000, base[:1000]]
            packed = b""
            for piece in pieces:
                packed += encoder.compress(piece) + encoder.flush(zlib.Z_SYNC_FLUSH)
            packed += encoder.flush()
            invoke(packed, b"".join(pieces), framing, 19, 1)
            invoke(packed, b"".join(pieces), framing, 1, 127)

            # Explicit maximum-distance reference crossing the history ring wrap.
            prefix = bytes(range(256)) * 128
            bits = Bits(); bits.put(3, 3); bits.code(197, 8); bits.code(29, 5)
            bits.put(8191, 13); bits.code(0, 7)
            raw = b"\x00\x00\x80\xff\x7f" + prefix + bits.finish()[2:-4]
            plain = prefix + prefix[:258]
            wrapped = b"\x78\x01" + raw + zlib.adler32(plain).to_bytes(4, "big")
            assert zlib.decompress(raw, -15) == plain and zlib.decompress(wrapped) == plain
            invoke(wrapped if framing == "zlib" else raw, plain, framing, 1, 1)
            if framing == "zlib":
                # Same valid DEFLATE, but a declared 256-byte window cannot serve it.
                invoke(b"\x08\x1d" + wrapped[2:], framing=framing, reject=True)

            raw = b"\x00\x00\x00\xff\xff" * 64 + b"\x01\x00\x00\xff\xff"
            packed = raw if framing == "raw" else b"\x78\x01" + raw + b"\x00\x00\x00\x01"
            invoke(packed, b"", framing, 1, 1)

            # Mutation oracle also handles valid altered raw streams (no checksum).
            original = zlib.compress(b"mutation stream" * 20)
            original = original if framing == "zlib" else original[2:-4]
            for _ in range(96):
                candidate = bytearray(original)
                candidate[random_source.randrange(len(candidate))] ^= 1 << random_source.randrange(8)
                candidate = bytes(candidate)
                oracle = zlib.decompressobj(window)
                try:
                    plain = oracle.decompress(candidate, 65537)
                    valid = oracle.eof and len(plain) <= 65536
                except zlib.error:
                    valid = False
                if valid:
                    boundary = len(candidate) - len(oracle.unused_data)
                    invoke(candidate[:boundary], plain, framing, 1, 1, max_output=65536, suffix=oracle.unused_data)
                else:
                    invoke(candidate, framing=framing, inchunk=1, outchunk=1, max_output=65536, reject=True)

        # Wrapped checksum corruption after output was produced remains an error.
        packed = zlib.compress(b"quarantined output" * 300)
        for index in range(len(packed) - 4, len(packed)):
            for bit in range(8):
                candidate = bytearray(packed); candidate[index] ^= 1 << bit
                invoke(bytes(candidate), inchunk=1, outchunk=1, reject=True)
        dictionary = zlib.compressobj(zdict=b"dictionary")
        invoke(dictionary.compress(b"dictionary") + dictionary.flush(), inchunk=1, outchunk=1, reject=True)
        for lengths in [[1, 1, 1, 1], [2, 2, 0, 0], [0, 0, 0, 0]]:
            bits = Bits(); bits.put(5, 3); bits.put(0, 14)
            for length in lengths: bits.put(length, 3)
            invoke(bits.finish(), inchunk=1, outchunk=1, reject=True)
        invoke(dynamic_literal(b"A", missing_end=True), inchunk=1, outchunk=1, reject=True)
    print(f"PASS {checks} incremental raw/zlib, split/truncation, backpressure, history, budget and mutation cases", flush=True)


if __name__ == "__main__":
    import sys
    check(sys.argv[1])