dymokomi/luce-std
Source of release 0.7.2
0.7.2 / tests / utf8_decoding / reference.py
#!/usr/bin/env python3
"""Write scalars.bin, malformed.bin and stream.bin into the directory given: UTF-8
references from Python's strict codec, fixed replay seed, which main.luc hands to
check.lucb and stream.lucb."""
from pathlib import Path
import random
import struct
import sys
OUTPUT = Path(sys.argv[1])
# All scalar values, including unassigned values and noncharacters. Surrogates
# have no UTF-8 representation and are tested as refused inputs in the program.
reference = OUTPUT / "scalars.bin"
reference.write_bytes("".join(chr(n) for n in range(0x110000)
if not 0xD800 <= n <= 0xDFFF).encode("utf-8"))
corpus = bytearray()
stream_corpus = bytearray()
def add(data):
try:
data.decode("utf-8", errors="strict")
invalid = 255
except UnicodeDecodeError as failure:
invalid = failure.start
corpus.extend((len(data), invalid))
corpus.extend(data)
results = [0xFFFFFFFF] * len(data)
failure_at, failure_kind = 255, 0
decoded_count = 0
for index, byte in enumerate(data):
try:
decoded = data[:index + 1].decode("utf-8", errors="strict")
except UnicodeDecodeError as failure:
# Final decoding distinguishes an incomplete valid prefix from an
# impossible prefix. CPython's incremental decoder can defer the
# ED A0 surrogate-prefix error until a third byte arrives.
if failure.reason == "unexpected end of data":
continue
failure_at, failure_kind = index, 1
break
assert len(decoded) == decoded_count + 1
results[index] = ord(decoded[-1])
decoded_count += 1
else:
try:
data.decode("utf-8", errors="strict")
except UnicodeDecodeError:
failure_at, failure_kind = len(data), 2
stream_corpus.extend((len(data), failure_at, failure_kind))
stream_corpus.extend(data)
for scalar in results:
stream_corpus.extend(struct.pack("<I", scalar))
add(b"")
for first in range(256):
add(bytes((first,)))
for second in range(256):
add(bytes((first, second)))
# Probe continuation limits at every position of three- and four-byte forms,
# including truncation and a preceding ASCII byte to check error offsets.
for scalar in (0x800, 0xD7FF, 0xE000, 0xFFFF, 0x10000, 0x10FFFF):
encoded = chr(scalar).encode("utf-8")
for index in range(len(encoded)):
add(encoded[:index])
for replacement in range(256):
mutated = encoded[:index] + bytes((replacement,)) + encoded[index + 1:]
add(mutated)
add(b"x" + mutated)
randomizer = random.Random(3629)
for _ in range(10000):
add(bytes(randomizer.randrange(256) for _ in range(randomizer.randrange(9))))
corpus_path = OUTPUT / "malformed.bin"
corpus_path.write_bytes(corpus)
stream_path = OUTPUT / "stream.bin"
stream_path.write_bytes(stream_corpus)