Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-compress

Source of release 0.9.0

0.9.0 / tests / check_gzip.py

#!/usr/bin/env python3
"""gzip against Python's gzip and zlib: members at every level with every header flag,
multi-member files with zero padding, decoded whole and in seeded pieces to the same
bytes; damaged files are accepted exactly when an RFC 1952 reading over zlib accepts
them (which Python's gzip then reads the same), with its output."""
import gzip
import random
import struct
import subprocess
import sys
import tempfile
import zlib
from pathlib import Path
from check_brotli import batch, sample


def member(rng, data):
    """A member with random header fields (FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT)."""
    flags = rng.randrange(32)
    header = bytearray(b"\x1f\x8b\x08" + bytes([flags]) + struct.pack("<I", rng.randrange(1 << 32)) + bytes([rng.choice([0, 2, 4]), rng.randrange(256)]))
    if flags & 4:
        extra = rng.randbytes(rng.randrange(0, 300))
        header += struct.pack("<H", len(extra)) + extra
    if flags & 8:
        header += bytes(rng.randrange(1, 256) for _ in range(rng.randrange(0, 60))) + b"\0"
    if flags & 16:
        header += bytes(rng.randrange(1, 256) for _ in range(rng.randrange(0, 200))) + b"\0"
    if flags & 2:
        header += struct.pack("<H", zlib.crc32(header) & 0xFFFF)
    body = zlib.compressobj(rng.randrange(10), zlib.DEFLATED, -15, rng.randrange(1, 10),
                            rng.choice([zlib.Z_DEFAULT_STRATEGY, zlib.Z_FILTERED, zlib.Z_HUFFMAN_ONLY, zlib.Z_RLE, zlib.Z_FIXED]))
    return bytes(header) + body.compress(data) + body.flush() + struct.pack("<II", zlib.crc32(data), len(data) & 0xFFFFFFFF)


def reference(data):
    """The file's data by RFC 1952 over zlib's raw inflate, or None when it is not a valid
    gzip file: a header CRC or reserved flag Python's gzip module ignores also fails. What
    it accepts Python's gzip must read the same."""
    out, at, members = bytearray(), 0, 0
    while members == 0 or any(data[at:]):
        if len(data) - at < 10 or data[at:at + 3] != b"\x1f\x8b\x08" or data[at + 3] & 0xE0:
            return None
        flags, end = data[at + 3], at + 10
        if flags & 4:
            if len(data) - end < 2:
                return None
            end += 2 + struct.unpack_from("<H", data, end)[0]
        for flag in (8, 16):
            if flags & flag:
                zero = data.find(b"\0", end)
                if zero < 0:
                    return None
                end = zero + 1
        if flags & 2:
            if end + 2 > len(data) or struct.unpack_from("<H", data, end)[0] != zlib.crc32(data[at:end]) & 0xFFFF:
                return None
            end += 2
        if end > len(data):
            return None
        inflater = zlib.decompressobj(-15)
        try:
            body = inflater.decompress(data[end:])
        except zlib.error:
            return None
        rest = inflater.unused_data
        if not inflater.eof or len(rest) < 8 or struct.unpack_from("<II", rest) != (zlib.crc32(body), len(body) & 0xFFFFFFFF):
            return None
        out += body
        at = len(data) - len(rest) + 8
        members += 1
    assert gzip.decompress(data) == out
    return bytes(out)


def check(tool, cases=300):
    tool = Path(tool).resolve()
    rng = random.Random(1952)
    jobs, expected, files = [], [], []
    for case in range(cases):
        members = [sample(rng, rng.choice([0, 1, 100, 5000, 40000, 200000])) for _ in range(rng.choice([1, 1, 1, 2, 3]))]
        packed = b"".join(member(rng, data) for data in members) + bytes(rng.choice([0, 0, 0, 1, 64]))
        data = b"".join(members)
        files.append(packed)
        piece, room = rng.choice([1, 3, 64, 4093, 1 << 20]), rng.choice([1, 7, 4096, 1 << 20])
        for mode in ["gzip", f"gzip/{piece}/{room}"]:
            if mode != "gzip" and len(data) > 60000 and min(piece, room) < 64:
                continue
            jobs.append((mode, packed))
            expected.append(f"OK {len(packed)} {len(data)} {zlib.crc32(data)}")
    for _ in range(cases * 3):
        damaged = bytearray(rng.choice(files))
        operation = rng.randrange(4)
        if operation == 0:
            damaged[rng.randrange(len(damaged))] ^= 1 << rng.randrange(8)
        elif operation == 1:
            del damaged[rng.randrange(len(damaged)):]
        elif operation == 2:
            damaged[rng.randrange(len(damaged) + 1):0] = rng.randbytes(rng.randrange(1, 9))
        else:
            at = rng.randrange(len(damaged))
            damaged[at:at + 4] = rng.randbytes(4)
        damaged = bytes(damaged)
        output = reference(damaged)
        jobs.append(("gzip", damaged))
        expected.append(None if output is None else f"OK {len(damaged)} {len(output)} {zlib.crc32(output)}")
    with tempfile.TemporaryDirectory(prefix="luce-compress-gzip-") as temporary:
        verdicts = batch(tool, jobs, Path(temporary))
    mismatches = 0
    for (mode, packed), want, got in zip(jobs, expected, verdicts):
        ok = got.startswith("ERR corrupt") if want is None else got == want
        if not ok:
            mismatches += 1
            print(f"MISMATCH {mode} {len(packed)} bytes: expected {want}, got {got}", flush=True)
    assert mismatches == 0, mismatches
    print(f"PASS {len(jobs)} gzip checks against Python's gzip", flush=True)


if __name__ == "__main__":
    check(sys.argv[1])