Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-bmp

Source of release 0.3.0

0.3.0 / tools / fuzz.py

#!/usr/bin/env python3
"""Mutate bitmaps and decode them: the decoder must never trap, hang or grow without
bound, whatever the bytes.

Each case is one of the corpus files (see conformance.py) changed by a few random
mutations: bytes flipped, inserted, deleted or repeated, the file cut short, chunks
spliced in from another file, and BMP's own values (header sizes, bit depths,
compressions, masks, RLE escapes, 32-bit fields set to 0 or 0xFFFFFFFF) dropped in at
random places. A fifth of the cases lose their file header and are read as bare DIBs,
half of those as icon DIBs with an AND mask. Cases run in batches through build/bmpcheck
(tools/bmpcheck.lucb, with `qs`: decoded, nothing written, a 4-megapixel limit), each
batch under a time limit; a batch that dies or times out is narrowed to the case
responsible, which is saved under build/fuzz-failures. The peak memory of the decoding
processes is reported.

First, a stress phase decodes generated bitmaps that are large or adversarial: 2048 x
2048 pixels of 32-bit bitfields, RLE8 data of single-pixel runs and of deltas, a stream
of row ends, a header claiming 65535 x 65535 pixels, and an icon DIB with its mask. Each
must finish in under two seconds.

  python3 tools/fuzz.py [--cases 20000] [--seed 1] [--guard-malloc]
"""
import argparse, os, random, resource, shutil, struct, subprocess, sys, tempfile, time
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tools"))
from conformance import CORPORA  # noqa: E402

TOOL = ROOT / "build/bmpcheck"
FAILURES = ROOT / "build/fuzz-failures"
# With --guard-malloc, the tool runs under macOS's Guard Malloc (every allocation on its
# own page, freed pages unmapped), which turns an overrun or a use after free into a crash.
ENVIRONMENT = dict(os.environ)
TOKENS = [b"BM", b"\x0c\x00\x00\x00", b"\x28\x00\x00\x00", b"\x40\x00\x00\x00", b"\x6c\x00\x00\x00",
          b"\x7c\x00\x00\x00", b"\x01\x00", b"\x02\x00", b"\x04\x00", b"\x08\x00", b"\x10\x00", b"\x18\x00",
          b"\x20\x00", b"\x01\x00\x00\x00", b"\x02\x00\x00\x00", b"\x03\x00\x00\x00", b"\x04\x00\x00\x00",
          b"\x06\x00\x00\x00", b"\xff\xff\xff\xff", b"\x00\x00\x00\x00", b"\x00\x00\x00\xff", b"\xff\xff\x00\x00",
          b"\x00\x00", b"\x00\x01", b"\x00\x02\xff\xff", b"\x00\xff", b"\xff\x00", b"\x00" * 64, b"\xff" * 64]


def bmp(width, height, bits, pixels, compression=0, palette=b"", masks=b"", header=40):
    """A BMP file with a BITMAPINFOHEADER (or V5) of the given fields."""
    info = struct.pack("<IiiHHIIiiII", header, width, height, 1, bits, compression, 0, 2835, 2835, 0, 0)
    info += masks + b"\x00" * (header - len(info) - len(masks))
    offset = 14 + len(info) + len(palette)
    return b"BM" + struct.pack("<IHHI", offset + len(pixels), 0, 0, offset) + info + palette + pixels


def stress_files():
    """Large and adversarial bitmaps: (name, bytes)."""
    masks = struct.pack("<IIII", 0x00FF0000, 0x0000FF00, 0x000000FF, 0xFF000000)
    yield "bitfields 2048x2048", bmp(2048, 2048, 32, os.urandom(2048 * 2048 * 4), 3, masks=masks, header=124)
    palette = bytes(range(256)) * 4
    runs = b"".join(b"\x01" + bytes([x % 256]) for x in range(2048)) + b"\x00\x00"
    yield "RLE8 single runs", bmp(2048, 2048, 8, runs * 2048 + b"\x00\x01", 1, palette=palette)
    yield "RLE8 deltas", bmp(2048, 2048, 8, b"\x00\x02\xff\x00" * 400000 + b"\x00\x02\x00\x01" * 400000, 1, palette=palette)
    yield "row ends", bmp(16, 2048, 8, b"\x00\x00" * 1000000, 1, palette=palette)
    yield "65535 x 65535", bmp(65535, 65535, 24, b"\x00" * 1000)
    yield "icon", bmp(2048, 4096, 32, b"\x00" * (2048 * 2048 * 4) + b"\x55" * (2048 * 2048 // 8))[14:], "qsi"


def stress():
    """Decode the stress files; answer the failures."""
    failures = []
    with tempfile.TemporaryDirectory() as directory:
        for name, data, *flags in stress_files():
            path = Path(directory) / "stress.bmp"
            path.write_bytes(data)
            started = time.monotonic()
            done, crashed, trailer = run([(flags[0] if flags else "qs", str(path))], timeout=30)
            spent = time.monotonic() - started
            verdict = "trap" if crashed else ("slow" if spent > 2 else "ok")
            print(f"stress {name:<20} {len(data) / 1e6:6.2f} MB {spent:6.2f} s  {verdict}", flush=True)
            if verdict != "ok":
                failures.append(f"stress {name}: {verdict} {trailer.strip()[-200:]}")
    return failures


def mutate(data, rng, donors):
    data = bytearray(data)
    for _ in range(rng.randint(1, 6)):
        choice = rng.random()
        at = rng.randint(0, len(data))
        if choice < 0.2 and data:
            data[min(at, len(data) - 1)] ^= 1 << rng.randint(0, 7)
        elif choice < 0.35:
            data[at:at] = rng.choice(TOKENS)
        elif choice < 0.45 and len(data) > 4:
            token = rng.choice(TOKENS)
            spot = rng.randint(0, min(len(data), 140) - 1)
            data[spot:spot + len(token)] = token
        elif choice < 0.55 and data:
            del data[at:at + rng.randint(1, 16)]
        elif choice < 0.62 and data:
            del data[at:]
        elif choice < 0.72 and data:
            end = min(len(data), at + rng.randint(1, 64))
            data[at:at] = data[at:end] * rng.randint(1, 8)
        elif choice < 0.85:
            donor = rng.choice(donors)
            start = rng.randint(0, max(0, len(donor) - 1))
            data[at:at] = donor[start:start + rng.randint(1, 200)]
        elif data:
            data[min(at, len(data) - 1)] = rng.choice([0, 1, 2, 3, 4, 5, 6, 8, 12, 16, 24, 32, 40, 0x80, 0xff])
    return bytes(data)


def run(jobs, timeout):
    """Run `jobs` [(flags, path)] in one tool process: (finished count, crashed?, stderr)."""
    with tempfile.TemporaryDirectory() as directory:
        listing = Path(directory) / "list.txt"
        listing.write_text("".join(f"{flags} {path}\n" for flags, path in jobs))
        try:
            result = subprocess.run([str(TOOL), str(listing), directory], capture_output=True, timeout=timeout, env=ENVIRONMENT)
        except subprocess.TimeoutExpired as expired:
            return len((expired.stdout or b"").splitlines()), True, "timeout"
        return len(result.stdout.splitlines()), result.returncode != 0, result.stderr.decode("utf-8", "replace")


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--cases", type=int, default=20000)
    parser.add_argument("--seed", type=int, default=1)
    parser.add_argument("--batch", type=int, default=500)
    parser.add_argument("--no-stress", action="store_true")
    parser.add_argument("--guard-malloc", action="store_true")
    options = parser.parse_args()
    if options.guard_malloc:
        ENVIRONMENT["DYLD_INSERT_LIBRARIES"] = "/usr/lib/libgmalloc.dylib"
    stressed = [] if options.no_stress else stress()
    rng = random.Random(options.seed)
    seeds = sorted(p for corpus in CORPORA.values() for p in corpus.rglob("*.bmp") if p.stat().st_size < 400000)
    donors = [p.read_bytes() for p in seeds]
    failures = []
    done, crashed, trailer = run([("qs", str(seed)) for seed in seeds], timeout=600)
    print(f"corpus: {len(seeds)} files, {'crashed: ' + trailer.strip()[-300:] if crashed else 'no traps'}", flush=True)
    if crashed:
        failures.append(f"corpus file {seeds[done]}: {trailer.strip()[-300:]}")
    with tempfile.TemporaryDirectory() as work:
        made = 0
        while made < options.cases:
            count = min(options.batch, options.cases - made)
            jobs = []
            for index in range(count):
                path = Path(work) / f"{made + index}.bmp"
                data, flags = mutate(rng.choice(donors), rng, donors), "qs"
                if rng.random() < 0.2:
                    data, flags = data[14:], rng.choice(["qsd", "qsi"])
                path.write_bytes(data)
                jobs.append((flags, str(path)))
            start = 0
            while start < len(jobs):
                done, crashed, trailer = run(jobs[start:], timeout=120)
                if not crashed:
                    break
                bad = start + done
                FAILURES.mkdir(parents=True, exist_ok=True)
                saved = FAILURES / f"case-{made + bad}-{jobs[bad][0]}.bmp"
                shutil.copy(jobs[bad][1], saved)
                failures.append(f"{saved}: {trailer.strip()[-300:]}")
                print(f"FAIL {failures[-1]}", flush=True)
                start = bad + 1
            made += count
            for path in Path(work).iterdir():
                path.unlink()
            print(f"{made} cases, {len(failures)} failing", flush=True)
    peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
    peak_mb = peak / (1024 * 1024) if sys.platform == "darwin" else peak / 1024
    print(f"{options.cases} cases: {len(failures)} traps or hangs; peak memory of a decoding process {peak_mb:.1f} MB")
    for line in stressed:
        print(f"FAIL {line}")
    return 1 if failures or stressed else 0


if __name__ == "__main__":
    sys.exit(main())