dymokomi/luce-ply
Source of release 0.1.0
## A GeometrySet as PLY, the counterpart of reader.lucb. The mesh, or else
## the point cloud, becomes the `vertex` element (and the mesh's faces the
## `face` element with `vertex_indices`); detail arrays named
## `element.property` become further elements. Attributes map back as the
## reader maps them: `N` to `nx ny nz`, `Cd` to `red green blue` (uchar)
## with `Alpha` as `alpha`, `uv` to `s t` (corner `uv` to a face `texcoord`
## list), other attributes to properties of their name (`name_0`, `name_1`,
## ... for tuples) and array attributes to lists.
##
## A cloud with the raw 3DGS names (reader.lucb) is written in the layout of
## the reference implementation: `x y z nx ny nz f_dc_0..2 f_rest_*
## opacity scale_0..2 rot_0..3`, all float, `f_rest` channel-major again, so
## a 3DGS file read and written comes back byte for byte.
##
## Binary rows without lists are encoded on the parallel pool; rows with
## lists, and ASCII, row by row. ASCII floats are written with the fewest
## digits that read back to the same value.
import memory
import strings
from luce_std import parallel
from luce_geocore.geocore import Mesh, Vector3
from luce_geocore.core.growing import Growing
from luce_geocore.kernel import Float3, Attribute, AttributeStore, Domain, DataType, GeometrySet, PointCloud, set_mesh, set_detail, points_component, mesh_component, cloud_positions, cloud_origin, cloud_store
from header import Format
from scalars import Scalar, scalar_name, size_of, load, load32, store, store32
from body import Lane
from warnings import Warnings
from reader import rest_name
## Where a list property's rows come from.
enum ListSource as u8:
## An array attribute: each element's items, every component.
ragged = 0
## A mesh's faces: each face's point indices.
faces = 1
## A corner attribute's first two components over each face's corners.
corners = 2
## One property being written: its PLY type and where each row's value
## comes from. A scalar reads `source + row * stride` as `lane`, adds `add`
## and multiplies by `scale` (no source: 0). A list reads its rows from
## `attribute` or the mesh's faces.
struct Field:
var label: u8[80]
var label_length: usize = 0
var kind: Scalar = Scalar.float32
var source: const u8*? = none
var lane: Lane = Lane.float32
var stride: usize = 0
var add: f64 = 0.0
var scale: f64 = 1.0
var is_list: bool = false
var count_kind: Scalar = Scalar.uint8
var list_source: ListSource = ListSource.ragged
var attribute: const Attribute*? = none
var mesh: const Mesh*? = none
func name() -> str:
return (str)self.label[..<self.label_length]
## Whether the value is the source's f32 bits unchanged.
func plain_float() -> bool:
return self.kind == Scalar.float32 and self.lane == Lane.float32 and self.add == 0.0 and self.scale == 1.0
## A field named `name` (and `_suffix` when `suffix` is not negative).
func field_named(name: str, suffix: i64) -> Field:
var field = Field()
memory.copy(field.label[..<name.length], name.bytes, name.length)
field.label_length = name.length
if suffix >= 0:
field.label[field.label_length] = '_'
field.label[field.label_length + 1] = (u8)(48 + suffix)
field.label_length += 2
return field
## A scalar field reading component `component` of `attribute`.
func attribute_field(name: str, suffix: i64, attribute: const Attribute*, component: usize, kind: Scalar, scale: f64) -> Field:
var field = field_named(name, suffix)
let bytes = attribute_lane_bytes(attribute.kind)
field.kind = kind
field.source = attribute.data.view().data + component * bytes
field.lane = attribute_lane(attribute.kind)
field.stride = attribute.width * bytes
field.scale = scale
return field
func attribute_lane(kind: DataType) -> Lane:
match kind:
.int8: return Lane.int8
.int32: return Lane.int32
.int64: return Lane.int64
.float64: return Lane.float64
_: return Lane.float32
func attribute_lane_bytes(kind: DataType) -> usize:
match kind:
.int8: return 1
.int64, .float64: return 8
_: return 4
## The PLY type an attribute of `kind` is written as.
func ply_kind(kind: DataType) -> Scalar:
match kind:
.int8: return Scalar.int8
.int32: return Scalar.int32
.float32: return Scalar.float32
_: return Scalar.float64
## The value of a scalar field at `row`.
inline func value_at(field: const Field*, row: usize) -> f64:
let source = field.source else return 0.0
let at = (const void*)(source + row * field.stride)
var value: f64 = 0.0
match field.lane:
.float32: value = (f64)memory.read[f32](at)
.float64: value = memory.read[f64](at)
.int8: value = (f64)memory.read[i8](at)
.int32: value = (f64)memory.read[i32](at)
_: value = (f64)memory.read[i64](at)
return (value + field.add) * field.scale
## How many values a list field has at `row`.
func list_length(field: const Field*, row: usize) -> usize:
match field.list_source:
.faces, .corners:
let mesh = field.mesh else return 0
let offsets = mesh.offset_span()
let corners = (usize)(offsets[row + 1] - offsets[row])
return corners * 2 if field.list_source == ListSource.corners else corners
_:
let attribute = field.attribute else return 0
return attribute.item_count(row) * attribute.width
## Value `index` of a list field's row `row`.
func list_value(field: const Field*, row: usize, index: usize) -> f64:
match field.list_source:
.faces:
let mesh = field.mesh else return 0.0
return (f64)mesh.corner_span()[(usize)mesh.offset_span()[row] + index]
.corners:
let mesh = field.mesh else return 0.0
let attribute = field.attribute else return 0.0
let corner = (usize)mesh.offset_span()[row] + index // 2
return attribute.number(corner, index % 2)
_:
let attribute = field.attribute else return 0.0
return attribute.item(row, index // attribute.width, index % attribute.width)
# mark: Elements
## One element being written: its name, rows and fields.
struct Output:
var label: u8[80]
var label_length: usize = 0
var rows: usize = 0
var fields: Growing[Field]
func name() -> str:
return (str)self.label[..<self.label_length]
func has_lists() -> bool:
for field in self.fields.view():
if field.is_list: return true
return false
func close():
self.fields.close()
func output_named(name: str, rows: usize) -> Output:
var output = Output(rows = rows)
memory.copy(output.label[..<name.length], name.bytes, name.length)
output.label_length = name.length
return output
## The elements of a set, closed together.
struct Plan:
var outputs: Growing[Output]
func close():
for at in 0..<self.outputs.count: self.outputs.items[at].close()
self.outputs.close()
## The bytes of `geometry` as a PLY file of `format`, on memory.heap (the
## caller frees them); what was left out is added to `warnings`.
pub func encode(geometry: const GeometrySet*, format: Format, warnings: Warnings*) -> u8[]!:
var plan = Plan()
defer plan.close()
try plan_set(geometry, &plan, warnings)
var bytes = Growing[u8]()
errdefer bytes.close()
try write_header(&plan, format, &bytes)
for at in 0..<plan.outputs.count:
try write_body(&plan.outputs.items[at], format, &bytes)
if bytes.count == bytes.items.length: return bytes.items
let exact = try new u8[bytes.count] --- in memory.heap
memory.copy(exact, bytes.items[..<bytes.count], bytes.count)
bytes.close()
return exact
func plan_set(geometry: const GeometrySet*, plan: Plan*, warnings: Warnings*) -> !:
let mesh = set_mesh(geometry)
let cloud_data = geometry.find(points_component())
for index in 0..<geometry.component_count():
let kind = geometry.component(index).kind
if kind == mesh_component() or kind == points_component(): continue
try warnings.add("Left out components PLY has no form for (instances, curves, volumes, CAD)")
if let found = mesh:
if cloud_data != none: try warnings.add("Left out the point cloud: PLY holds one vertex element, the mesh's")
try plan_mesh(found, plan, warnings)
elif let data = cloud_data:
try plan_cloud((const PointCloud*)data, plan, warnings)
try plan_details(set_detail(geometry), plan, warnings)
## Positions as x, y, z: float when the origin is zero (exact), else double.
func position_fields(positions: const Float3[], origin: Vector3, output: Output*) -> !:
let names: str[3] = ["x", "y", "z"]
let adds: f64[3] = [origin.x, origin.y, origin.z]
let wide = origin.x != 0.0 or origin.y != 0.0 or origin.z != 0.0
for lane in 0..<(usize)3:
var field = field_named(names[lane], -1)
field.kind = Scalar.float64 if wide else Scalar.float32
field.source = (const u8*)positions.data + lane * 4
field.stride = 12
field.add = adds[lane]
try output.fields.push(field)
func plan_mesh(mesh: const Mesh*, plan: Plan*, warnings: Warnings*) -> !:
# The plan owns the outputs from the start, so a failure closes them.
let first = plan.outputs.count
try add_output(plan, output_named("vertex", (usize)mesh.point_count()))
try add_output(plan, output_named("face", (usize)mesh.face_count()))
let points = &plan.outputs.items[first]
let faces = &plan.outputs.items[first + 1]
try position_fields(mesh.point_span(), mesh.position_origin(), points)
var indices = field_named("vertex_indices", -1)
indices.is_list = true
indices.kind = Scalar.int32
indices.list_source = ListSource.faces
indices.mesh = mesh
var largest: i32 = 0
let offsets = mesh.offset_span()
for face in 0..<offsets.length - 1:
if offsets[face + 1] - offsets[face] > largest: largest = offsets[face + 1] - offsets[face]
indices.count_kind = Scalar.uint8 if largest <= 127 else Scalar.int32
try faces.fields.push(indices)
for index in 0..<mesh.attribute_count():
let attribute = try mesh.attribute_entry(index)
if attribute.domain == Domain.point:
try plan_attribute(attribute, points, warnings)
elif attribute.domain == Domain.face:
try plan_attribute(attribute, faces, warnings)
elif attribute.domain == Domain.corner and attribute.name() == "uv" and attribute.width >= 2 and not attribute.is_ragged() and attribute.kind != DataType.boolean and attribute.kind != DataType.text:
var texcoord = field_named("texcoord", -1)
texcoord.is_list = true
texcoord.kind = Scalar.float32
texcoord.list_source = ListSource.corners
texcoord.mesh = mesh
texcoord.attribute = attribute
texcoord.count_kind = Scalar.uint8 if largest <= 127 else Scalar.int32
try faces.fields.push(texcoord)
elif attribute.domain == Domain.corner or attribute.domain == Domain.edge:
try warnings.add("Left out corner and edge attributes PLY has no form for (corner uv is written as texcoord)")
func plan_cloud(cloud: const PointCloud*, plan: Plan*, warnings: Warnings*) -> !:
let positions = cloud_positions(cloud)
let attributes = cloud_store(cloud)
try add_output(plan, output_named("vertex", positions.length))
let points = &plan.outputs.items[plan.outputs.count - 1]
try position_fields(positions, cloud_origin(cloud), points)
var used = Growing[bool]()
defer used.close()
try used.fill(attributes.count(), false)
if try plan_splats(attributes, used.items[..<attributes.count()], points, warnings):
for index in 0..<attributes.count():
if not used.items[index]: try plan_attribute(attributes.entry(index), points, warnings)
else:
for index in 0..<attributes.count(): try plan_attribute(attributes.entry(index), points, warnings)
func add_output(plan: Plan*, output: Output) -> !:
var owned = output
plan.outputs.push(owned) catch failure:
owned.close()
error(failure.code, failure.message)
## Index of float point attribute `name` of width `width` in `attributes`, or -1.
func float_attribute(attributes: const AttributeStore*, name: str, width: usize) -> i64:
let index = attributes.find(name, Domain.point)
if index < 0: return -1
let entry = attributes.entry((usize)index)
if entry.is_ragged() or entry.kind != DataType.float32 or entry.width != width: return -1
return index
## The 3DGS layout for a cloud with the raw splat names; false (nothing
## planned) without them. Marks the attributes it writes in `used`.
func plan_splats(attributes: const AttributeStore*, used: bool[], points: Output*, warnings: Warnings*) -> bool!:
let leading: str[4] = ["f_dc_0", "f_dc_1", "f_dc_2", ""]
let trailing: str[8] = ["opacity", "scale_0", "scale_1", "scale_2", "rot_0", "rot_1", "rot_2", "rot_3"]
for name in leading[..<3]:
if float_attribute(attributes, name, 1) < 0: return false
for name in trailing:
if float_attribute(attributes, name, 1) < 0: return false
# Normals: N when there, else the zeros the reference writes.
let normal = float_attribute(attributes, "N", 3)
let normal_names: str[3] = ["nx", "ny", "nz"]
for lane in 0..<(usize)3:
if normal >= 0:
try points.fields.push(attribute_field(normal_names[lane], -1, attributes.entry((usize)normal), lane, Scalar.float32, 1.0))
else:
try points.fields.push(field_named(normal_names[lane], -1))
if normal >= 0: used[(usize)normal] = true
for name in leading[..<3]:
let index = (usize)float_attribute(attributes, name, 1)
try points.fields.push(attribute_field(name, -1, attributes.entry(index), 0, Scalar.float32, 1.0))
used[index] = true
let rest = attributes.find("f_rest", Domain.point)
if rest >= 0:
let entry = attributes.entry((usize)rest)
let per_channel = entry.item_count(0) if points.rows > 0 else 0
var uniform = entry.is_ragged() and entry.kind == DataType.float32 and entry.width == 3 and (per_channel == 3 or per_channel == 8 or per_channel == 15)
if uniform:
for row in 0..<points.rows: uniform = uniform and entry.item_count(row) == per_channel
if uniform:
used[(usize)rest] = true
var name_buffer: u8[16]
let values = entry.data.view().data
for index in 0..<per_channel * 3:
let channel = index // per_channel
let coefficient = index % per_channel
var field = field_named(rest_name(name_buffer, index), -1)
field.source = values + (coefficient * 3 + channel) * 4
field.stride = per_channel * 12
try points.fields.push(field)
else:
try warnings.add("Wrote f_rest as a list: its items are not 3, 8 or 15 per point")
for name in trailing:
let index = (usize)float_attribute(attributes, name, 1)
try points.fields.push(attribute_field(name, -1, attributes.entry(index), 0, Scalar.float32, 1.0))
used[index] = true
return true
## The properties of one attribute on `output`'s element.
func plan_attribute(attribute: const Attribute*, output: Output*, warnings: Warnings*) -> !:
if attribute.kind == DataType.boolean or attribute.kind == DataType.text:
try warnings.add("Left out groups and text attributes, which PLY has no form for")
return
if attribute.elements != output.rows:
try warnings.add("Left out attributes with one value for all elements")
return
let name = attribute.name()
if attribute.is_ragged():
var field = field_named(name, -1)
field.is_list = true
field.kind = Scalar.float32 if attribute.kind == DataType.float32 else Scalar.int32
field.list_source = ListSource.ragged
field.attribute = attribute
var largest: usize = 0
for row in 0..<attribute.elements:
let count = attribute.item_count(row) * attribute.width
if count > largest: largest = count
field.count_kind = Scalar.uint8 if largest <= 255 else Scalar.int32
try output.fields.push(field)
return
if name == "N" and attribute.width == 3:
let names: str[3] = ["nx", "ny", "nz"]
for lane in 0..<(usize)3: try output.fields.push(attribute_field(names[lane], -1, attribute, lane, ply_kind(attribute.kind), 1.0))
return
if name == "Cd" and attribute.width >= 3 and not integer_kind(attribute.kind):
let names: str[4] = ["red", "green", "blue", "alpha"]
for lane in 0..<attribute.width: try output.fields.push(attribute_field(names[lane], -1, attribute, lane, Scalar.uint8, 255.0))
return
if name == "Alpha" and attribute.width == 1 and not integer_kind(attribute.kind):
try output.fields.push(attribute_field("alpha", -1, attribute, 0, Scalar.uint8, 255.0))
return
if name == "uv" and attribute.width >= 2:
try output.fields.push(attribute_field("s", -1, attribute, 0, ply_kind(attribute.kind), 1.0))
try output.fields.push(attribute_field("t", -1, attribute, 1, ply_kind(attribute.kind), 1.0))
return
if attribute.kind == DataType.int64: try warnings.add("Wrote 64-bit integer attributes as double, PLY's widest type")
for lane in 0..<attribute.width:
try output.fields.push(attribute_field(name, (i64)lane if attribute.width > 1 else -1, attribute, lane, ply_kind(attribute.kind), 1.0))
func integer_kind(kind: DataType) -> bool:
return kind == DataType.int8 or kind == DataType.int32 or kind == DataType.int64
## Detail arrays `element.property` (one element each) as elements: the
## arrays of one element name in order, each a property, all of one length.
func plan_details(detail: const AttributeStore*, plan: Plan*, warnings: Warnings*) -> !:
for index in 0..<detail.count():
let entry = detail.entry(index)
let dot = strings.find(entry.name(), ".") else continue
if not entry.is_ragged() or entry.width != 1 or entry.elements != 1 or dot == 0: continue
let element_name = (str)entry.name().bytes[..<dot]
if element_name == "vertex" or element_name == "face": continue
let property_name = (str)entry.name().bytes[dot + 1..]
let rows = entry.item_count(0)
var found: i64 = -1
for at in 0..<plan.outputs.count:
if plan.outputs.items[at].name() == element_name: found = (i64)at
if found < 0:
try add_output(plan, output_named(element_name, rows))
found = (i64)plan.outputs.count - 1
let output = &plan.outputs.items[(usize)found]
if output.rows != rows:
try warnings.add("Left out detail arrays whose length differs from the rest of their element")
continue
var field = field_named(property_name, -1)
field.kind = Scalar.float32 if entry.kind == DataType.float32 else Scalar.int32
field.lane = Lane.float32 if entry.kind == DataType.float32 else Lane.int32
field.source = entry.data.view().data
field.stride = 4
try output.fields.push(field)
# mark: Text and bytes
func put_text(bytes: Growing[u8]*, text: str) -> !:
try bytes.reserve(bytes.count + text.length)
memory.copy(bytes.items[bytes.count..<bytes.count + text.length], text.bytes, text.length)
bytes.count += text.length
func write_header(plan: const Plan*, format: Format, bytes: Growing[u8]*) -> !:
var line: u8[256]
try put_text(bytes, "ply\n")
match format:
.ascii: try put_text(bytes, "format ascii 1.0\n")
.little_endian: try put_text(bytes, "format binary_little_endian 1.0\n")
_: try put_text(bytes, "format binary_big_endian 1.0\n")
for output in plan.outputs.view():
try put_text(bytes, try strings.format(line, f"element {output.name()} {output.rows}\n"))
for field in output.fields.view():
if field.is_list:
try put_text(bytes, try strings.format(line, f"property list {scalar_name(field.count_kind)} {scalar_name(field.kind)} {field.name()}\n"))
else:
try put_text(bytes, try strings.format(line, f"property {scalar_name(field.kind)} {field.name()}\n"))
try put_text(bytes, "end_header\n")
func write_body(output: const Output*, format: Format, bytes: Growing[u8]*) -> !:
if format == Format.ascii:
try write_ascii(output, bytes)
return
let swap = format == Format.big_endian
if output.has_lists():
try write_rows(output, swap, bytes)
return
var stride: usize = 0
for field in output.fields.view(): stride += size_of(field.kind)
let total = stride * output.rows
try bytes.reserve(bytes.count + total)
var work = FixedWrite(fields = output.fields.view(), stride = stride, rows = bytes.items.data + bytes.count, swap = swap)
try parallel.for_each_range(output.rows, 1024, (void*)&work, write_block)
bytes.count += total
struct FixedWrite:
var fields: const Field[]
var stride: usize
var rows: u8*
var swap: bool
func write_block(context: void*, first: usize, end: usize, worker: usize):
_ = worker
let work = (FixedWrite*)context
var offset: usize = 0
for index in 0..<work.fields.length:
let field = &work.fields[index]
let target = work.rows + offset
let row_bytes = work.stride
offset += size_of(field.kind)
if let source = field.source:
if field.plain_float():
# Bits are copied as they are, NaN payloads included.
for row in first..<end:
store32(target + row * row_bytes, load32(source + row * field.stride, false), work.swap)
continue
for row in first..<end:
store(target + row * row_bytes, field.kind, work.swap, value_at(field, row))
func write_rows(output: const Output*, swap: bool, bytes: Growing[u8]*) -> !:
var scratch: u8[8]
for row in 0..<output.rows:
for at in 0..<output.fields.count:
let field = &output.fields.view()[at]
if not field.is_list:
try put_value(bytes, scratch, field.kind, swap, value_at(field, row))
continue
let count = list_length(field, row)
try put_value(bytes, scratch, field.count_kind, swap, (f64)count)
for index in 0..<count: try put_value(bytes, scratch, field.kind, swap, list_value(field, row, index))
func put_value(bytes: Growing[u8]*, scratch: u8[], kind: Scalar, swap: bool, value: f64) -> !:
store(scratch.data, kind, swap, value)
let width = size_of(kind)
for at in 0..<width: try bytes.push(scratch[at])
func write_ascii(output: const Output*, bytes: Growing[u8]*) -> !:
var digits: u8[64]
for row in 0..<output.rows:
var first = true
for at in 0..<output.fields.count:
let field = &output.fields.view()[at]
if not field.is_list:
if not first: try bytes.push(' ')
try put_number(bytes, digits, field.kind, value_at(field, row))
first = false
continue
let count = list_length(field, row)
if not first: try bytes.push(' ')
try put_number(bytes, digits, field.count_kind, (f64)count)
first = false
for index in 0..<count:
try bytes.push(' ')
try put_number(bytes, digits, field.kind, list_value(field, row, index))
try bytes.push(10)
## `value` as text for a property of `kind`: shortest round-trip digits for
## floats, the stored integer otherwise.
func put_number(bytes: Growing[u8]*, digits: u8[], kind: Scalar, value: f64) -> !:
if kind == Scalar.float32:
try put_text(bytes, try strings.format_f32((f32)value, digits))
return
if kind == Scalar.float64:
try put_text(bytes, try strings.format_f64(value, digits))
return
var stored: u8[8]
store(&stored[0], kind, false, value)
try put_text(bytes, try strings.format_i64((i64)load(&stored[0], kind, false), digits))