dymokomi/luce-git
Source of release 0.5.2
0.5.2 / tests / oracles / pack_oracle.py
"""Native pack reader versus stock Git, with hostile complete pack fixtures."""
import hashlib
from pathlib import Path
import struct
import subprocess
import tempfile
import zlib
from delta_oracle import varint, pack_header, copy_instruction, identity
def finish(entries, count, version=2):
wire = b'PACK' + struct.pack('>II', version, count) + entries
return wire + hashlib.sha1(wire).digest()
def ofs(distance):
data = bytearray([distance & 127])
while distance >> 7:
distance = (distance >> 7) - 1
data.insert(0, 128 | (distance & 127))
return bytes(data)
def check(binary):
with tempfile.TemporaryDirectory(prefix='git-pack-') as temporary:
root = Path(temporary)
path = root / 'input.pack'
bases_path = root / 'bases.pack'
def run(wire, expected=None, bases=None, wrong=False):
path.write_bytes(wire)
arguments = [str(binary), str(path)]
if bases is not None:
bases_path.write_bytes(bases)
arguments.append(str(bases_path))
if wrong: arguments.append('wrong')
result = subprocess.run(arguments, capture_output=True, timeout=30)
assert result.returncode >= 0 and b'Sanitizer' not in result.stderr and b'runtime error:' not in result.stderr, result.stderr
if expected is None:
assert result.returncode != 0, 'invalid pack accepted'
else:
assert result.returncode == 0, result.stderr
assert sorted(result.stdout.decode().splitlines()) == sorted(expected), result.stdout
def line(payload):
return f'blob {identity(payload).hex()} {len(payload)}'
base = b'base source\n' * 100
target = base + b'new'
delta = varint(len(base)) + varint(len(target)) + copy_instruction(0, len(base)) + b'\x03new'
full = pack_header(3, len(base)) + zlib.compress(base)
ref = pack_header(7, len(delta)) + identity(base) + zlib.compress(delta)
offset = pack_header(6, len(delta)) + ofs(len(full)) + zlib.compress(delta)
expected = [line(base), line(target)]
run(finish(full + ref, 2), expected)
run(finish(ref + full, 2), expected) # forward ref base
run(finish(full + offset, 2, 3), expected)
run(finish(b'', 0), [])
mixed, mixed_expected = b'', []
for form, kind in enumerate(('commit', 'tree', 'blob', 'tag'), 1):
payload = b'' if kind == 'tree' else kind.encode() + b' fixture\n'
mixed += pack_header(form, len(payload)) + zlib.compress(payload)
oid = hashlib.sha1(kind.encode() + b' ' + str(len(payload)).encode() + b'\0' + payload).hexdigest()
mixed_expected.append(f'{kind} {oid} {len(payload)}')
run(finish(mixed, 4), mixed_expected)
run(finish(ref, 1)) # thin/missing base
run(finish(ref, 1), [line(target)], bases=finish(full, 1))
run(finish(ref, 1), bases=finish(b'', 0))
wrong_base = pack_header(3, 5) + zlib.compress(b'wrong')
run(finish(ref, 1), bases=finish(wrong_base, 1), wrong=True)
# Forward internal dependency must become resolvable after an external
# base lookup, even when the first callback returns missing.
later = target + b'!'
later_delta = varint(len(target)) + varint(len(later)) + copy_instruction(0, len(target)) + b'\x01!'
later_ref = pack_header(7, len(later_delta)) + identity(target) + zlib.compress(later_delta)
run(finish(later_ref + ref, 2), [line(target), line(later)], bases=finish(full, 1))
run(finish(full + pack_header(6, len(delta)) + b'\0' + zlib.compress(delta), 2))
run(finish(full + pack_header(6, len(delta)) + ofs(len(full) - 1) + zlib.compress(delta), 2))
for version in (0, 1, 4): run(finish(b'', 0, version))
run(finish(b'', 65537))
run(finish(full, 0))
run(finish(full, 2))
run(finish(full + b'extra', 1))
run(finish(pack_header(0, 0) + zlib.compress(b''), 1))
run(finish(pack_header(5, 0) + zlib.compress(b''), 1))
run(finish(pack_header(3, len(base) - 1) + zlib.compress(base), 1))
run(finish(pack_header(3, len(base) + 1) + zlib.compress(base), 1))
run(finish(pack_header(3, 67108865), 1))
run(finish(pack_header(3, 0) + b'bad-zlib', 1))
valid = finish(full, 1)
for cut in range(len(valid)): run(valid[:cut])
run(valid[:-1] + bytes([valid[-1] ^ 1]))
# Chain depth is measured after resolution, including forward refs.
payload = b'x'
chain = pack_header(3, 1) + zlib.compress(payload)
chain_expected = [line(payload)]
for depth in range(1, 66):
next_payload = payload + b'x'
change = varint(len(payload)) + varint(len(next_payload)) + copy_instruction(0, len(payload)) + b'\x01x'
chain += pack_header(7, len(change)) + identity(payload) + zlib.compress(change)
payload = next_payload
chain_expected.append(line(payload))
if depth == 64: run(finish(chain, 65), chain_expected)
run(finish(chain, 66))
# Stock Git generates full objects and real delta chains in both forms.
repo = root / 'repo.git'
subprocess.run(['git', 'init', '--bare', '--quiet', str(repo)], check=True)
ids, expected = [], []
for i in range(24):
payload = b'common package source\n' * 200 + f'change {i}\n'.encode()
oid = subprocess.check_output(['git', '--git-dir', str(repo), 'hash-object', '-w', '--stdin'], input=payload).strip()
ids.append(oid)
expected.append(line(payload))
for flags in ([], ['--delta-base-offset']):
wire = subprocess.check_output(['git', '--git-dir', str(repo), 'pack-objects', '--stdout', '--window=24', '--depth=20', *flags], input=b'\n'.join(ids) + b'\n')
run(wire, expected)
# A large push: more objects than the old 4096 cap, most of them deltas, which
# resolve in a few passes rather than one base per pass.
many, many_expected = [], []
stock = b''.join(f'line {n} of a shared source file\n'.encode() for n in range(300))
for i in range(5000):
payload = stock + f'revision {i}\n'.encode()
many_expected.append(line(payload))
many.append(payload)
hashed = subprocess.run(['git', '--git-dir', str(repo), 'hash-object', '-w', '--stdin-paths'],
input='\n'.join(str(write_blob(root, i, payload)) for i, payload in enumerate(many)).encode(),
capture_output=True, check=True).stdout
wire = subprocess.check_output(['git', '--git-dir', str(repo), 'pack-objects', '--stdout', '--delta-base-offset'], input=hashed)
run(wire, many_expected)
print('PASS native packs: forward REF/OFS deltas, stock Git packs (5000 objects too), checksum/framing/limits and rejection', flush=True)
def write_blob(root, index, payload):
path = root / f'blob{index}'
path.write_bytes(payload)
return path