dymokomi/luce-spz
Source of release 0.1.0
## The SPZ container (Niantic's format, versions 1 to 4). Every SPZ holds the
## same six attribute streams, one after another, per splat in order:
##
## | Stream | Bytes per splat | Encoding |
## |---|---|---|
## | positions | 9 (v1: 6) | x, y, z as 24-bit signed fixed point, `fractional_bits` fractional (v1: f16) |
## | alphas | 1 | sigmoid(opacity) · 255 |
## | colors | 3 | f_dc · 0.15 · 255 + 127.5 |
## | scales | 3 | (ln σ + 10) · 16 |
## | rotations | 4 (v1-v2: 3) | smallest three of x, y, z, w: 2-bit index, 3 × (sign + 9-bit magnitude); v1-v2: x, y, z · 127.5 + 127.5 |
## | sh | 3 per coefficient | x · 128 + 128, coefficient-major, RGB per coefficient |
##
## Versions 1-3 are one gzip stream: a 16-byte header (magic `NGSP`, version,
## splat count, SH degree, fractional bits, flags, a reserved byte), the
## streams, then extension records when flag 2 is set. Version 4 is a 32-byte
## plain header (the same fields, then the stream count and the table of
## contents' offset), extension records up to that offset, a table of
## (compressed, uncompressed) u64 sizes, and each stream zstd-compressed on
## its own (an empty sh stream is left out). All values are little-endian.
## Splats are stored in RUB axes (x right, y up, z back) unless an extension
## names others.
import memory
import errors
## `NGSP` read as a little-endian u32.
pub let magic: u32 = 0x5053474e
## The newest version read and the one written.
pub let latest_version: u32 = 4
## Flag bits.
pub let antialiased_flag: u8 = 1
pub let extensions_flag: u8 = 2
## The largest SH degree a file holds (24 coefficients), and the largest kept
## (luce-geocore's `sh` holds bands 1 to 3).
pub let max_file_degree: usize = 4
pub let max_kept_degree: usize = 3
## Header sizes.
pub let legacy_header_bytes: usize = 16
pub let header_bytes: usize = 32
## A header claiming more splats than this many per byte of file is refused
## before anything is allocated (the reference's bound: a 1024:1 ratio over
## 9 bytes per splat).
let max_ratio: u64 = 1024
let min_bytes_per_splat: u64 = 9
## SH coefficients per color channel of `degree`: 0, 3, 8, 15 or 24.
pub func sh_dimension(degree: usize) -> usize:
return (degree + 1) * (degree + 1) - 1
## The fields of either header.
pub struct Header:
pub var version: u32
pub var splats: usize
pub var degree: usize
pub var fractional_bits: u32
pub var flags: u8
## Version 4 only: the zstd streams and where their table starts.
pub var streams: usize
pub var table: usize
## Whether positions are f16 (version 1).
pub func half_positions() -> bool:
return self.version == 1
## Whether rotations are the smallest three in 4 bytes (versions 3 and up).
pub func smallest_three() -> bool:
return self.version >= 3
## The bytes of stream `stream` (0..<6, in the table's order).
pub func stream_bytes(stream: usize) -> usize:
let n = self.splats
return match stream:
0 => n * (6 if self.half_positions() else 9)
1 => n
2 => n * 3
3 => n * 3
4 => n * (4 if self.smallest_three() else 3)
_ => n * sh_dimension(self.degree) * 3
## The six streams' bytes together.
pub func body_bytes() -> usize:
var total: usize = 0
for stream in 0..<(usize)6: total += self.stream_bytes(stream)
return total
## A little-endian u32 at `at` (the caller checked the length).
pub func read_u32(data: const u8[], at: usize) -> u32:
return (u32)data[at] | ((u32)data[at + 1] << 8) | ((u32)data[at + 2] << 16) | ((u32)data[at + 3] << 24)
## A little-endian u64 at `at`.
pub func read_u64(data: const u8[], at: usize) -> u64:
return (u64)read_u32(data, at) | ((u64)read_u32(data, at + 4) << 32)
## Write `value` little-endian at `at`.
pub func write_u32(data: u8[], at: usize, value: u32):
for k in 0..<(usize)4: data[at + k] = (u8)(value >> (u32)(k * 8))
pub func write_u64(data: u8[], at: usize, value: u64):
for k in 0..<(usize)8: data[at + k] = (u8)(value >> (u64)(k * 8))
## The fields shared by both headers (magic to flags), checked: a known
## version, a splat count `size` bytes can hold, an SH degree up to 4.
func common_fields(data: const u8[], size: usize, legacy: bool) -> Header!:
if read_u32(data, 0) != magic: error(errors.invalid, "not an SPZ file (no NGSP header)")
var header: Header
header.version = read_u32(data, 4)
let splats = read_u32(data, 8)
header.degree = (usize)data[12]
header.fractional_bits = (u32)data[13]
header.flags = data[14]
if header.version < 1 or header.version > latest_version:
error(errors.unsupported, "SPZ versions 1 to 4 are read")
if legacy and header.version >= 4: error(errors.invalid, "a version 4 SPZ is not gzip-compressed")
if not legacy and header.version < 4: error(errors.invalid, "an SPZ before version 4 is gzip-compressed")
if splats == 0 or splats > 0x7FFFFFFF or (u64)splats > (u64)size * max_ratio // min_bytes_per_splat:
error(errors.invalid, "the SPZ header's splat count does not fit the file")
header.splats = (usize)splats
if header.degree > max_file_degree: error(errors.invalid, "SPZ SH degrees are 0 to 4")
if header.fractional_bits > 23: error(errors.invalid, "SPZ positions have at most 23 fractional bits")
return header
## The 16-byte header at the start of a version 1-3 file's decompressed bytes.
pub func read_legacy_header(data: const u8[]) -> Header!:
if data.length < legacy_header_bytes: error(errors.invalid, "the SPZ stream is shorter than its header")
let header = try common_fields(data, data.length, true)
if data.length - legacy_header_bytes < header.body_bytes():
error(errors.invalid, "the SPZ stream is shorter than its splats")
return header
## The 32-byte header of a version 4 file of `data`.
pub func read_header(data: const u8[]) -> Header!:
if data.length < header_bytes: error(errors.invalid, "the SPZ file is shorter than its header")
var header = try common_fields(data, data.length, false)
header.streams = (usize)data[15]
header.table = (usize)read_u32(data, 16)
if header.table < header_bytes or header.table > data.length or (data.length - header.table) // 16 < header.streams:
error(errors.invalid, "the SPZ table of contents is outside the file")
let expected: usize = 6 if header.degree > 0 else 5
if header.streams != expected: error(errors.invalid, "an SPZ has 6 streams, or 5 without SH")
return header
## Write `header`'s first 16 bytes (magic to the reserved byte) into `out`.
pub func write_common(out: u8[], header: const Header*):
write_u32(out, 0, magic)
write_u32(out, 4, header.version)
write_u32(out, 8, (u32)header.splats)
out[12] = (u8)header.degree
out[13] = (u8)header.fractional_bits
out[14] = header.flags
out[15] = 0