Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-ico

Source of release 0.3.1

0.3.1 / tools / fuzz.py

#!/usr/bin/env python3
"""Mutate icons and cursors and decode every entry: the decoder must never trap, hang or
grow without bound, whatever the bytes.

Each case is one of the corpus files (see conformance.py) changed by a few random
mutations: bytes flipped, inserted, deleted or repeated, the file cut short, chunks
spliced in from another file, and the format's own values (types, counts, sizes of 0 and
256, offsets and lengths set to 0 or 0xFFFFFFFF, a PNG signature, DIB header sizes and
depths) dropped in at random places, the directory most often. Cases run in batches
through build/icocheck (tools/icocheck.lucb, with `qs`: every entry read, chosen and
decoded, nothing written, a 4-megapixel limit), each batch under a time limit; a batch
that dies or times out is narrowed to the case responsible, which is saved under
build/fuzz-failures. The peak memory of the decoding processes is reported.

First, a stress phase reads generated files that are large or adversarial: a directory
of 65535 entries sharing one image, entries whose offsets and lengths point past the
file, PNG entries whose IHDR claims 4 billion pixels a side, and a 2048 x 2048 32-bit
entry with its mask. Each must finish in under two seconds.

  python3 tools/fuzz.py [--cases 20000] [--seed 1] [--guard-malloc]
"""
import argparse, os, random, resource, shutil, struct, subprocess, sys, tempfile, time
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tools"))
from conformance import CORPORA  # noqa: E402

TOOL = ROOT / "build/icocheck"
FAILURES = ROOT / "build/fuzz-failures"
# With --guard-malloc, the tool runs under macOS's Guard Malloc (every allocation on its
# own page, freed pages unmapped), which turns an overrun or a use after free into a crash.
ENVIRONMENT = dict(os.environ)
TOKENS = [b"\x00\x00\x01\x00", b"\x00\x00\x02\x00", b"\x01\x00", b"\xff\xff", b"\x00", b"\x10", b"\x20", b"\x00\x00",
          b"\x00\x00\x00\x00", b"\xff\xff\xff\xff", b"\x16\x00\x00\x00", b"\x28\x00\x00\x00", b"\x0c\x00\x00\x00",
          b"\x89PNG\r\n\x1a\n", b"IHDR", b"\x01\x00\x20\x00", b"\x01\x00\x08\x00", b"\x01\x00\x01\x00",
          b"\x00\x00\x00\x80", b"\x00" * 64, b"\xff" * 64]


def icon(entries, images, kind=1):
    """An icon of `entries` [(width, height, planes, bits)] and their `images` bytes."""
    offset = 6 + 16 * len(entries)
    directory, body = b"", b""
    for (width, height, planes, bits), data in zip(entries, images):
        directory += struct.pack("<BBBBHHII", width % 256, height % 256, 0, 0, planes, bits, len(data), offset + len(body))
        body += data
    return struct.pack("<HHH", 0, kind, len(entries)) + directory + body


def dib(width, height, pixel=b"\x10\x20\x30\x00", mask=b"\x55"):
    header = struct.pack("<IiiHHIIiiII", 40, width, height * 2, 1, 32, 0, 0, 0, 0, 0, 0)
    return header + pixel * (width * height) + mask * ((width + 31) // 32 * 4 * height)


def stress_files():
    """Large and adversarial icons: (name, bytes)."""
    image = dib(16, 16)
    directory = b"".join(struct.pack("<BBBBHHII", 16, 16, 0, 0, 1, 32, len(image), 6 + 16 * 65535) for _ in range(65535))
    yield "65535 entries", struct.pack("<HHH", 0, 1, 65535) + directory + image
    far = b"".join(struct.pack("<BBBBHHII", 0, 0, 0, 0, 1, 32, 0xFFFFFFFF, 0xFFFFE000 + n) for n in range(5000))
    yield "offsets past the end", struct.pack("<HHH", 0, 1, 5000) + far
    head = b"\x89PNG\r\n\x1a\n\x00\x00\x00\x0dIHDR\xff\xff\xff\xff\xff\xff\xff\xff\x08\x06\x00\x00\x00"
    yield "huge PNG entries", icon([(0, 0, 1, 32)] * 1000, [head] * 1000)
    yield "2048 x 2048 entry", icon([(0, 0, 1, 32)], [dib(2048, 2048)])


def stress():
    """Decode the stress files; answer the failures."""
    failures = []
    with tempfile.TemporaryDirectory() as directory:
        for name, data, *flags in stress_files():
            path = Path(directory) / "stress.ico"
            path.write_bytes(data)
            started = time.monotonic()
            done, crashed, trailer = run([(flags[0] if flags else "qs", str(path))], timeout=30)
            spent = time.monotonic() - started
            verdict = "trap" if crashed else ("slow" if spent > 2 else "ok")
            print(f"stress {name:<20} {len(data) / 1e6:6.2f} MB {spent:6.2f} s  {verdict}", flush=True)
            if verdict != "ok":
                failures.append(f"stress {name}: {verdict} {trailer.strip()[-200:]}")
    return failures


def mutate(data, rng, donors):
    data = bytearray(data)
    for _ in range(rng.randint(1, 6)):
        choice = rng.random()
        at = rng.randint(0, len(data))
        if choice < 0.2 and data:
            data[min(at, len(data) - 1)] ^= 1 << rng.randint(0, 7)
        elif choice < 0.3:
            data[at:at] = rng.choice(TOKENS)
        elif choice < 0.5 and len(data) > 4:
            token = rng.choice(TOKENS)
            spot = rng.randint(0, min(len(data), 6 + 16 * 8) - 1)
            data[spot:spot + len(token)] = token
        elif choice < 0.58 and data:
            del data[at:at + rng.randint(1, 16)]
        elif choice < 0.65 and data:
            del data[at:]
        elif choice < 0.75 and data:
            end = min(len(data), at + rng.randint(1, 64))
            data[at:at] = data[at:end] * rng.randint(1, 8)
        elif choice < 0.88:
            donor = rng.choice(donors)
            start = rng.randint(0, max(0, len(donor) - 1))
            data[at:at] = donor[start:start + rng.randint(1, 200)]
        elif data:
            data[min(at, len(data) - 1)] = rng.choice([0, 1, 2, 4, 8, 16, 24, 32, 40, 0x80, 0xff])
    return bytes(data)


def run(jobs, timeout):
    """Run `jobs` [(flags, path)] in one tool process: (finished count, crashed?, stderr)."""
    with tempfile.TemporaryDirectory() as directory:
        listing = Path(directory) / "list.txt"
        listing.write_text("".join(f"{flags} {path}\n" for flags, path in jobs))
        try:
            result = subprocess.run([str(TOOL), str(listing), directory], capture_output=True, timeout=timeout, env=ENVIRONMENT)
        except subprocess.TimeoutExpired as expired:
            return len((expired.stdout or b"").splitlines()), True, "timeout"
        return len(result.stdout.splitlines()), result.returncode != 0, result.stderr.decode("utf-8", "replace")


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--cases", type=int, default=20000)
    parser.add_argument("--seed", type=int, default=1)
    parser.add_argument("--batch", type=int, default=500)
    parser.add_argument("--no-stress", action="store_true")
    parser.add_argument("--guard-malloc", action="store_true")
    options = parser.parse_args()
    if options.guard_malloc:
        ENVIRONMENT["DYLD_INSERT_LIBRARIES"] = "/usr/lib/libgmalloc.dylib"
    stressed = [] if options.no_stress else stress()
    rng = random.Random(options.seed)
    seeds = sorted({p for folders in CORPORA.values() for folder in folders for pattern in ("*.ico", "*.cur")
                    for p in folder.glob(pattern) if p.stat().st_size < 400000})
    donors = [p.read_bytes() for p in seeds]
    failures = []
    done, crashed, trailer = run([("qs", str(seed)) for seed in seeds], timeout=600)
    print(f"corpus: {len(seeds)} files, {'crashed: ' + trailer.strip()[-300:] if crashed else 'no traps'}", flush=True)
    if crashed:
        failures.append(f"corpus file {seeds[done]}: {trailer.strip()[-300:]}")
    with tempfile.TemporaryDirectory() as work:
        made = 0
        while made < options.cases:
            count = min(options.batch, options.cases - made)
            jobs = []
            for index in range(count):
                path = Path(work) / f"{made + index}.ico"
                path.write_bytes(mutate(rng.choice(donors), rng, donors))
                jobs.append(("qs", str(path)))
            start = 0
            while start < len(jobs):
                done, crashed, trailer = run(jobs[start:], timeout=120)
                if not crashed:
                    break
                bad = start + done
                FAILURES.mkdir(parents=True, exist_ok=True)
                saved = FAILURES / f"case-{made + bad}.ico"
                shutil.copy(jobs[bad][1], saved)
                failures.append(f"{saved}: {trailer.strip()[-300:]}")
                print(f"FAIL {failures[-1]}", flush=True)
                start = bad + 1
            made += count
            for path in Path(work).iterdir():
                path.unlink()
            print(f"{made} cases, {len(failures)} failing", flush=True)
    peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
    peak_mb = peak / (1024 * 1024) if sys.platform == "darwin" else peak / 1024
    print(f"{options.cases} cases: {len(failures)} traps or hangs; peak memory of a decoding process {peak_mb:.1f} MB")
    for line in stressed:
        print(f"FAIL {line}")
    return 1 if failures or stressed else 0


if __name__ == "__main__":
    sys.exit(main())