dymokomi/luce-ply
Source of release 0.1.0
## Element bodies: each property's values go to a Sink. A scalar property's
## sink is a strided column (an attribute's lane, the positions); a list
## property's sink keeps per-row counts and the items. Binary elements
## without lists have a fixed row size and decode on the parallel pool, a
## block of rows per task and a property at a time within it (the 3DGS fast
## path); elements with lists and ASCII bodies decode row by row.
import memory
import strings
from luce_std import parallel
from luce_geocore.core.growing import Growing
from scalars import Scalar, size_of, is_float, load, load32, load64
from header import Header, Format, Property
import errors
## The type a sink column stores.
pub enum Lane as u8:
float32 = 0
float64 = 1
int8 = 2
int32 = 3
int64 = 4
## Where one property's values go. A scalar property with `data` stores
## each row's value v as (v - offset) * scale at data + row * stride; one
## without data is skipped. A list property with `keep` appends each row's
## count to `counts` and its items to `floats` (float items) or `integers`.
pub struct Sink:
pub var data: u8*? = none
pub var lane: Lane = Lane.float32
pub var stride: usize = 0
pub var scale: f64 = 1.0
pub var offset: f64 = 0.0
pub var keep: bool = false
pub var counts: Growing[u32]
pub var floats: Growing[f32]
pub var integers: Growing[i32]
pub func close():
self.counts.close()
self.floats.close()
self.integers.close()
## Store `value` into a column of `lane` at `at`.
inline func put(at: u8*, lane: Lane, value: f64):
match lane:
.float32: memory.write[f32]((void*)at, (f32)value)
.float64: memory.write[f64]((void*)at, value)
.int8: memory.write[i8]((void*)at, (i8)value)
.int32: memory.write[i32]((void*)at, (i32)value)
_: memory.write[i64]((void*)at, (i64)value)
## Decode element `element` from `data` at `at` into `sinks` (one per
## property of the element), returning where the next element starts.
pub func decode_element(header: const Header*, element: usize, data: const u8[], at: usize, sinks: Sink[]) -> usize!:
let properties = header.properties_of(element)
if sinks.length != properties.length: error(errors.invalid, "one sink per property")
let shape = &header.elements.items[element]
if header.format == Format.ascii:
return try decode_ascii(properties, shape.count, data, at, sinks)
let swap = header.format == Format.big_endian
if shape.stride > 0:
let bytes = (u64)shape.stride * (u64)shape.count
if (u64)(data.length - at) < bytes: error(errors.invalid, "the PLY body is shorter than its header says")
try decode_fixed(properties, shape.stride, shape.count, data.data + at, swap, sinks)
return at + (usize)bytes
return try decode_rows(properties, shape.count, data, at, swap, sinks)
# mark: Fixed rows, in parallel
## Rows per pool task: about a quarter of a megabyte of a 3DGS body.
let block_rows: usize = 1024
struct FixedWork:
var properties: const Property[]
var offsets: const usize[]
var stride: usize
var rows: const u8*
var swap: bool
var sinks: Sink[]
func decode_fixed(properties: const Property[], stride: usize, count: usize, rows: const u8*, swap: bool, sinks: Sink[]) -> !:
var offsets = Growing[usize]()
defer offsets.close()
var offset: usize = 0
for property in properties:
try offsets.push(offset)
offset += size_of(property.kind)
var work = FixedWork(properties = properties, offsets = offsets.view(), stride = stride, rows = rows, swap = swap, sinks = sinks)
try parallel.for_each_range(count, block_rows, (void*)&work, fixed_block)
func fixed_block(context: void*, first: usize, end: usize, worker: usize):
_ = worker
let work = (FixedWork*)context
for index in 0..<work.properties.length:
let sink = &work.sinks[index]
let target = sink.data else continue
let kind = work.properties[index].kind
let source = work.rows + work.offsets[index]
let row_bytes = work.stride
let step = sink.stride
let plain = sink.scale == 1.0 and sink.offset == 0.0
if plain and kind == Scalar.float32 and sink.lane == Lane.float32:
# Bits are copied as they are, NaN payloads included.
for row in first..<end:
memory.write[u32]((void*)(target + row * step), load32(source + row * row_bytes, work.swap))
elif plain and kind == Scalar.float64 and sink.lane == Lane.float64:
for row in first..<end:
memory.write[u64]((void*)(target + row * step), load64(source + row * row_bytes, work.swap))
else:
let lane = sink.lane
let scale = sink.scale
let shift = sink.offset
for row in first..<end:
put(target + row * step, lane, (load(source + row * row_bytes, kind, work.swap) - shift) * scale)
# mark: Rows with lists
func decode_rows(properties: const Property[], count: usize, data: const u8[], start: usize, swap: bool, sinks: Sink[]) -> usize!:
var at = start
let bytes = data.data
for row in 0..<count:
for index in 0..<properties.length:
let property = &properties[index]
let sink = &sinks[index]
if not property.is_list:
let width = size_of(property.kind)
if data.length - at < width: error(errors.invalid, "the PLY body is shorter than its header says")
if let target = sink.data:
put(target + row * sink.stride, sink.lane, (load(bytes + at, property.kind, swap) - sink.offset) * sink.scale)
at += width
continue
let count_width = size_of(property.count_kind)
if data.length - at < count_width: error(errors.invalid, "the PLY body is shorter than its header says")
let items = try list_count(load(bytes + at, property.count_kind, swap))
at += count_width
let width = size_of(property.kind)
if (u64)(data.length - at) < (u64)items * (u64)width: error(errors.invalid, "the PLY body is shorter than its header says")
if sink.keep:
try sink.counts.push((u32)items)
for item in 0..<items:
try keep_item(sink, property.kind, load(bytes + at + item * width, property.kind, swap))
at += items * width
return at
## A list's count, checked.
func list_count(value: f64) -> usize!:
if not (value >= 0.0) or value > 16777216.0: error(errors.invalid, "a PLY list count is negative or too large")
return (usize)value
## Append one list item to the sink's items.
func keep_item(sink: Sink*, kind: Scalar, value: f64) -> !:
if is_float(kind):
try sink.floats.push((f32)value)
return
if value < -2147483648.0 or value > 2147483647.0: error(errors.limit, "a PLY list holds an integer past 32 bits")
try sink.integers.push((i32)value)
# mark: ASCII
## The number starting at the next token after `at`; `at` moves past it.
func next_number(data: const u8[], at: usize*) -> f64!:
var position = *at
while position < data.length and (data[position] == 32 or data[position] == 9 or data[position] == 10 or data[position] == 13):
position += 1
if position == data.length: error(errors.invalid, "the PLY body has fewer values than its header says")
let (value, used) = strings.scan_f64(data[position..]) catch failure:
_ = failure
error(errors.invalid, "a PLY ASCII value is not a number")
position += used
if position < data.length and not (data[position] == 32 or data[position] == 9 or data[position] == 10 or data[position] == 13):
error(errors.invalid, "a PLY ASCII value is not a number")
*at = position
return value
func decode_ascii(properties: const Property[], count: usize, data: const u8[], start: usize, sinks: Sink[]) -> usize!:
var at = start
for row in 0..<count:
for index in 0..<properties.length:
let property = &properties[index]
let sink = &sinks[index]
if not property.is_list:
let value = try next_number(data, &at)
if let target = sink.data:
put(target + row * sink.stride, sink.lane, (value - sink.offset) * sink.scale)
continue
let items = try list_count(try next_number(data, &at))
if sink.keep: try sink.counts.push((u32)items)
for item in 0..<items:
_ = item
let value = try next_number(data, &at)
if sink.keep: try keep_item(sink, property.kind, value)
return at