Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-ply

Source of release 0.1.0

0.1.0 / src / writer.lucb

## A GeometrySet as PLY, the counterpart of reader.lucb. The mesh, or else
## the point cloud, becomes the `vertex` element (and the mesh's faces the
## `face` element with `vertex_indices`); detail arrays named
## `element.property` become further elements. Attributes map back as the
## reader maps them: `N` to `nx ny nz`, `Cd` to `red green blue` (uchar)
## with `Alpha` as `alpha`, `uv` to `s t` (corner `uv` to a face `texcoord`
## list), other attributes to properties of their name (`name_0`, `name_1`,
## ... for tuples) and array attributes to lists.
##
## A cloud with the raw 3DGS names (reader.lucb) is written in the layout of
## the reference implementation: `x y z nx ny nz f_dc_0..2 f_rest_*
## opacity scale_0..2 rot_0..3`, all float, `f_rest` channel-major again, so
## a 3DGS file read and written comes back byte for byte.
##
## Binary rows without lists are encoded on the parallel pool; rows with
## lists, and ASCII, row by row. ASCII floats are written with the fewest
## digits that read back to the same value.
import memory
import strings
from luce_std import parallel
from luce_geocore.geocore import Mesh, Vector3
from luce_geocore.core.growing import Growing
from luce_geocore.kernel import Float3, Attribute, AttributeStore, Domain, DataType, GeometrySet, PointCloud, set_mesh, set_detail, points_component, mesh_component, cloud_positions, cloud_origin, cloud_store
from header import Format
from scalars import Scalar, scalar_name, size_of, load, load32, store, store32
from body import Lane
from warnings import Warnings
from reader import rest_name

## Where a list property's rows come from.
enum ListSource as u8:
    ## An array attribute: each element's items, every component.
    ragged = 0
    ## A mesh's faces: each face's point indices.
    faces = 1
    ## A corner attribute's first two components over each face's corners.
    corners = 2

## One property being written: its PLY type and where each row's value
## comes from. A scalar reads `source + row * stride` as `lane`, adds `add`
## and multiplies by `scale` (no source: 0). A list reads its rows from
## `attribute` or the mesh's faces.
struct Field:
    var label: u8[80]
    var label_length: usize = 0
    var kind: Scalar = Scalar.float32
    var source: const u8*? = none
    var lane: Lane = Lane.float32
    var stride: usize = 0
    var add: f64 = 0.0
    var scale: f64 = 1.0
    var is_list: bool = false
    var count_kind: Scalar = Scalar.uint8
    var list_source: ListSource = ListSource.ragged
    var attribute: const Attribute*? = none
    var mesh: const Mesh*? = none

    func name() -> str:
        return (str)self.label[..<self.label_length]

    ## Whether the value is the source's f32 bits unchanged.
    func plain_float() -> bool:
        return self.kind == Scalar.float32 and self.lane == Lane.float32 and self.add == 0.0 and self.scale == 1.0

## A field named `name` (and `_suffix` when `suffix` is not negative).
func field_named(name: str, suffix: i64) -> Field:
    var field = Field()
    memory.copy(field.label[..<name.length], name.bytes, name.length)
    field.label_length = name.length
    if suffix >= 0:
        field.label[field.label_length] = '_'
        field.label[field.label_length + 1] = (u8)(48 + suffix)
        field.label_length += 2
    return field

## A scalar field reading component `component` of `attribute`.
func attribute_field(name: str, suffix: i64, attribute: const Attribute*, component: usize, kind: Scalar, scale: f64) -> Field:
    var field = field_named(name, suffix)
    let bytes = attribute_lane_bytes(attribute.kind)
    field.kind = kind
    field.source = attribute.data.view().data + component * bytes
    field.lane = attribute_lane(attribute.kind)
    field.stride = attribute.width * bytes
    field.scale = scale
    return field

func attribute_lane(kind: DataType) -> Lane:
    match kind:
        .int8: return Lane.int8
        .int32: return Lane.int32
        .int64: return Lane.int64
        .float64: return Lane.float64
        _: return Lane.float32

func attribute_lane_bytes(kind: DataType) -> usize:
    match kind:
        .int8: return 1
        .int64, .float64: return 8
        _: return 4

## The PLY type an attribute of `kind` is written as.
func ply_kind(kind: DataType) -> Scalar:
    match kind:
        .int8: return Scalar.int8
        .int32: return Scalar.int32
        .float32: return Scalar.float32
        _: return Scalar.float64

## The value of a scalar field at `row`.
inline func value_at(field: const Field*, row: usize) -> f64:
    let source = field.source else return 0.0
    let at = (const void*)(source + row * field.stride)
    var value: f64 = 0.0
    match field.lane:
        .float32: value = (f64)memory.read[f32](at)
        .float64: value = memory.read[f64](at)
        .int8: value = (f64)memory.read[i8](at)
        .int32: value = (f64)memory.read[i32](at)
        _: value = (f64)memory.read[i64](at)
    return (value + field.add) * field.scale

## How many values a list field has at `row`.
func list_length(field: const Field*, row: usize) -> usize:
    match field.list_source:
        .faces, .corners:
            let mesh = field.mesh else return 0
            let offsets = mesh.offset_span()
            let corners = (usize)(offsets[row + 1] - offsets[row])
            return corners * 2 if field.list_source == ListSource.corners else corners
        _:
            let attribute = field.attribute else return 0
            return attribute.item_count(row) * attribute.width

## Value `index` of a list field's row `row`.
func list_value(field: const Field*, row: usize, index: usize) -> f64:
    match field.list_source:
        .faces:
            let mesh = field.mesh else return 0.0
            return (f64)mesh.corner_span()[(usize)mesh.offset_span()[row] + index]
        .corners:
            let mesh = field.mesh else return 0.0
            let attribute = field.attribute else return 0.0
            let corner = (usize)mesh.offset_span()[row] + index // 2
            return attribute.number(corner, index % 2)
        _:
            let attribute = field.attribute else return 0.0
            return attribute.item(row, index // attribute.width, index % attribute.width)

# mark: Elements

## One element being written: its name, rows and fields.
struct Output:
    var label: u8[80]
    var label_length: usize = 0
    var rows: usize = 0
    var fields: Growing[Field]

    func name() -> str:
        return (str)self.label[..<self.label_length]

    func has_lists() -> bool:
        for field in self.fields.view():
            if field.is_list: return true
        return false

    func close():
        self.fields.close()

func output_named(name: str, rows: usize) -> Output:
    var output = Output(rows = rows)
    memory.copy(output.label[..<name.length], name.bytes, name.length)
    output.label_length = name.length
    return output

## The elements of a set, closed together.
struct Plan:
    var outputs: Growing[Output]

    func close():
        for at in 0..<self.outputs.count: self.outputs.items[at].close()
        self.outputs.close()

## The bytes of `geometry` as a PLY file of `format`, on memory.heap (the
## caller frees them); what was left out is added to `warnings`.
pub func encode(geometry: const GeometrySet*, format: Format, warnings: Warnings*) -> u8[]!:
    var plan = Plan()
    defer plan.close()
    try plan_set(geometry, &plan, warnings)
    var bytes = Growing[u8]()
    errdefer bytes.close()
    try write_header(&plan, format, &bytes)
    for at in 0..<plan.outputs.count:
        try write_body(&plan.outputs.items[at], format, &bytes)
    if bytes.count == bytes.items.length: return bytes.items
    let exact = try new u8[bytes.count] --- in memory.heap
    memory.copy(exact, bytes.items[..<bytes.count], bytes.count)
    bytes.close()
    return exact

func plan_set(geometry: const GeometrySet*, plan: Plan*, warnings: Warnings*) -> !:
    let mesh = set_mesh(geometry)
    let cloud_data = geometry.find(points_component())
    for index in 0..<geometry.component_count():
        let kind = geometry.component(index).kind
        if kind == mesh_component() or kind == points_component(): continue
        try warnings.add("Left out components PLY has no form for (instances, curves, volumes, CAD)")
    if let found = mesh:
        if cloud_data != none: try warnings.add("Left out the point cloud: PLY holds one vertex element, the mesh's")
        try plan_mesh(found, plan, warnings)
    elif let data = cloud_data:
        try plan_cloud((const PointCloud*)data, plan, warnings)
    try plan_details(set_detail(geometry), plan, warnings)

## Positions as x, y, z: float when the origin is zero (exact), else double.
func position_fields(positions: const Float3[], origin: Vector3, output: Output*) -> !:
    let names: str[3] = ["x", "y", "z"]
    let adds: f64[3] = [origin.x, origin.y, origin.z]
    let wide = origin.x != 0.0 or origin.y != 0.0 or origin.z != 0.0
    for lane in 0..<(usize)3:
        var field = field_named(names[lane], -1)
        field.kind = Scalar.float64 if wide else Scalar.float32
        field.source = (const u8*)positions.data + lane * 4
        field.stride = 12
        field.add = adds[lane]
        try output.fields.push(field)

func plan_mesh(mesh: const Mesh*, plan: Plan*, warnings: Warnings*) -> !:
    # The plan owns the outputs from the start, so a failure closes them.
    let first = plan.outputs.count
    try add_output(plan, output_named("vertex", (usize)mesh.point_count()))
    try add_output(plan, output_named("face", (usize)mesh.face_count()))
    let points = &plan.outputs.items[first]
    let faces = &plan.outputs.items[first + 1]
    try position_fields(mesh.point_span(), mesh.position_origin(), points)
    var indices = field_named("vertex_indices", -1)
    indices.is_list = true
    indices.kind = Scalar.int32
    indices.list_source = ListSource.faces
    indices.mesh = mesh
    var largest: i32 = 0
    let offsets = mesh.offset_span()
    for face in 0..<offsets.length - 1:
        if offsets[face + 1] - offsets[face] > largest: largest = offsets[face + 1] - offsets[face]
    indices.count_kind = Scalar.uint8 if largest <= 127 else Scalar.int32
    try faces.fields.push(indices)
    for index in 0..<mesh.attribute_count():
        let attribute = try mesh.attribute_entry(index)
        if attribute.domain == Domain.point:
            try plan_attribute(attribute, points, warnings)
        elif attribute.domain == Domain.face:
            try plan_attribute(attribute, faces, warnings)
        elif attribute.domain == Domain.corner and attribute.name() == "uv" and attribute.width >= 2 and not attribute.is_ragged() and attribute.kind != DataType.boolean and attribute.kind != DataType.text:
            var texcoord = field_named("texcoord", -1)
            texcoord.is_list = true
            texcoord.kind = Scalar.float32
            texcoord.list_source = ListSource.corners
            texcoord.mesh = mesh
            texcoord.attribute = attribute
            texcoord.count_kind = Scalar.uint8 if largest <= 127 else Scalar.int32
            try faces.fields.push(texcoord)
        elif attribute.domain == Domain.corner or attribute.domain == Domain.edge:
            try warnings.add("Left out corner and edge attributes PLY has no form for (corner uv is written as texcoord)")

func plan_cloud(cloud: const PointCloud*, plan: Plan*, warnings: Warnings*) -> !:
    let positions = cloud_positions(cloud)
    let attributes = cloud_store(cloud)
    try add_output(plan, output_named("vertex", positions.length))
    let points = &plan.outputs.items[plan.outputs.count - 1]
    try position_fields(positions, cloud_origin(cloud), points)
    var used = Growing[bool]()
    defer used.close()
    try used.fill(attributes.count(), false)
    if try plan_splats(attributes, used.items[..<attributes.count()], points, warnings):
        for index in 0..<attributes.count():
            if not used.items[index]: try plan_attribute(attributes.entry(index), points, warnings)
    else:
        for index in 0..<attributes.count(): try plan_attribute(attributes.entry(index), points, warnings)

func add_output(plan: Plan*, output: Output) -> !:
    var owned = output
    plan.outputs.push(owned) catch failure:
        owned.close()
        error(failure.code, failure.message)

## Index of float point attribute `name` of width `width` in `attributes`, or -1.
func float_attribute(attributes: const AttributeStore*, name: str, width: usize) -> i64:
    let index = attributes.find(name, Domain.point)
    if index < 0: return -1
    let entry = attributes.entry((usize)index)
    if entry.is_ragged() or entry.kind != DataType.float32 or entry.width != width: return -1
    return index

## The 3DGS layout for a cloud with the raw splat names; false (nothing
## planned) without them. Marks the attributes it writes in `used`.
func plan_splats(attributes: const AttributeStore*, used: bool[], points: Output*, warnings: Warnings*) -> bool!:
    let leading: str[4] = ["f_dc_0", "f_dc_1", "f_dc_2", ""]
    let trailing: str[8] = ["opacity", "scale_0", "scale_1", "scale_2", "rot_0", "rot_1", "rot_2", "rot_3"]
    for name in leading[..<3]:
        if float_attribute(attributes, name, 1) < 0: return false
    for name in trailing:
        if float_attribute(attributes, name, 1) < 0: return false
    # Normals: N when there, else the zeros the reference writes.
    let normal = float_attribute(attributes, "N", 3)
    let normal_names: str[3] = ["nx", "ny", "nz"]
    for lane in 0..<(usize)3:
        if normal >= 0:
            try points.fields.push(attribute_field(normal_names[lane], -1, attributes.entry((usize)normal), lane, Scalar.float32, 1.0))
        else:
            try points.fields.push(field_named(normal_names[lane], -1))
    if normal >= 0: used[(usize)normal] = true
    for name in leading[..<3]:
        let index = (usize)float_attribute(attributes, name, 1)
        try points.fields.push(attribute_field(name, -1, attributes.entry(index), 0, Scalar.float32, 1.0))
        used[index] = true
    let rest = attributes.find("f_rest", Domain.point)
    if rest >= 0:
        let entry = attributes.entry((usize)rest)
        let per_channel = entry.item_count(0) if points.rows > 0 else 0
        var uniform = entry.is_ragged() and entry.kind == DataType.float32 and entry.width == 3 and (per_channel == 3 or per_channel == 8 or per_channel == 15)
        if uniform:
            for row in 0..<points.rows: uniform = uniform and entry.item_count(row) == per_channel
        if uniform:
            used[(usize)rest] = true
            var name_buffer: u8[16]
            let values = entry.data.view().data
            for index in 0..<per_channel * 3:
                let channel = index // per_channel
                let coefficient = index % per_channel
                var field = field_named(rest_name(name_buffer, index), -1)
                field.source = values + (coefficient * 3 + channel) * 4
                field.stride = per_channel * 12
                try points.fields.push(field)
        else:
            try warnings.add("Wrote f_rest as a list: its items are not 3, 8 or 15 per point")
    for name in trailing:
        let index = (usize)float_attribute(attributes, name, 1)
        try points.fields.push(attribute_field(name, -1, attributes.entry(index), 0, Scalar.float32, 1.0))
        used[index] = true
    return true

## The properties of one attribute on `output`'s element.
func plan_attribute(attribute: const Attribute*, output: Output*, warnings: Warnings*) -> !:
    if attribute.kind == DataType.boolean or attribute.kind == DataType.text:
        try warnings.add("Left out groups and text attributes, which PLY has no form for")
        return
    if attribute.elements != output.rows:
        try warnings.add("Left out attributes with one value for all elements")
        return
    let name = attribute.name()
    if attribute.is_ragged():
        var field = field_named(name, -1)
        field.is_list = true
        field.kind = Scalar.float32 if attribute.kind == DataType.float32 else Scalar.int32
        field.list_source = ListSource.ragged
        field.attribute = attribute
        var largest: usize = 0
        for row in 0..<attribute.elements:
            let count = attribute.item_count(row) * attribute.width
            if count > largest: largest = count
        field.count_kind = Scalar.uint8 if largest <= 255 else Scalar.int32
        try output.fields.push(field)
        return
    if name == "N" and attribute.width == 3:
        let names: str[3] = ["nx", "ny", "nz"]
        for lane in 0..<(usize)3: try output.fields.push(attribute_field(names[lane], -1, attribute, lane, ply_kind(attribute.kind), 1.0))
        return
    if name == "Cd" and attribute.width >= 3 and not integer_kind(attribute.kind):
        let names: str[4] = ["red", "green", "blue", "alpha"]
        for lane in 0..<attribute.width: try output.fields.push(attribute_field(names[lane], -1, attribute, lane, Scalar.uint8, 255.0))
        return
    if name == "Alpha" and attribute.width == 1 and not integer_kind(attribute.kind):
        try output.fields.push(attribute_field("alpha", -1, attribute, 0, Scalar.uint8, 255.0))
        return
    if name == "uv" and attribute.width >= 2:
        try output.fields.push(attribute_field("s", -1, attribute, 0, ply_kind(attribute.kind), 1.0))
        try output.fields.push(attribute_field("t", -1, attribute, 1, ply_kind(attribute.kind), 1.0))
        return
    if attribute.kind == DataType.int64: try warnings.add("Wrote 64-bit integer attributes as double, PLY's widest type")
    for lane in 0..<attribute.width:
        try output.fields.push(attribute_field(name, (i64)lane if attribute.width > 1 else -1, attribute, lane, ply_kind(attribute.kind), 1.0))

func integer_kind(kind: DataType) -> bool:
    return kind == DataType.int8 or kind == DataType.int32 or kind == DataType.int64

## Detail arrays `element.property` (one element each) as elements: the
## arrays of one element name in order, each a property, all of one length.
func plan_details(detail: const AttributeStore*, plan: Plan*, warnings: Warnings*) -> !:
    for index in 0..<detail.count():
        let entry = detail.entry(index)
        let dot = strings.find(entry.name(), ".") else continue
        if not entry.is_ragged() or entry.width != 1 or entry.elements != 1 or dot == 0: continue
        let element_name = (str)entry.name().bytes[..<dot]
        if element_name == "vertex" or element_name == "face": continue
        let property_name = (str)entry.name().bytes[dot + 1..]
        let rows = entry.item_count(0)
        var found: i64 = -1
        for at in 0..<plan.outputs.count:
            if plan.outputs.items[at].name() == element_name: found = (i64)at
        if found < 0:
            try add_output(plan, output_named(element_name, rows))
            found = (i64)plan.outputs.count - 1
        let output = &plan.outputs.items[(usize)found]
        if output.rows != rows:
            try warnings.add("Left out detail arrays whose length differs from the rest of their element")
            continue
        var field = field_named(property_name, -1)
        field.kind = Scalar.float32 if entry.kind == DataType.float32 else Scalar.int32
        field.lane = Lane.float32 if entry.kind == DataType.float32 else Lane.int32
        field.source = entry.data.view().data
        field.stride = 4
        try output.fields.push(field)

# mark: Text and bytes

func put_text(bytes: Growing[u8]*, text: str) -> !:
    try bytes.reserve(bytes.count + text.length)
    memory.copy(bytes.items[bytes.count..<bytes.count + text.length], text.bytes, text.length)
    bytes.count += text.length

func write_header(plan: const Plan*, format: Format, bytes: Growing[u8]*) -> !:
    var line: u8[256]
    try put_text(bytes, "ply\n")
    match format:
        .ascii: try put_text(bytes, "format ascii 1.0\n")
        .little_endian: try put_text(bytes, "format binary_little_endian 1.0\n")
        _: try put_text(bytes, "format binary_big_endian 1.0\n")
    for output in plan.outputs.view():
        try put_text(bytes, try strings.format(line, f"element {output.name()} {output.rows}\n"))
        for field in output.fields.view():
            if field.is_list:
                try put_text(bytes, try strings.format(line, f"property list {scalar_name(field.count_kind)} {scalar_name(field.kind)} {field.name()}\n"))
            else:
                try put_text(bytes, try strings.format(line, f"property {scalar_name(field.kind)} {field.name()}\n"))
    try put_text(bytes, "end_header\n")

func write_body(output: const Output*, format: Format, bytes: Growing[u8]*) -> !:
    if format == Format.ascii:
        try write_ascii(output, bytes)
        return
    let swap = format == Format.big_endian
    if output.has_lists():
        try write_rows(output, swap, bytes)
        return
    var stride: usize = 0
    for field in output.fields.view(): stride += size_of(field.kind)
    let total = stride * output.rows
    try bytes.reserve(bytes.count + total)
    var work = FixedWrite(fields = output.fields.view(), stride = stride, rows = bytes.items.data + bytes.count, swap = swap)
    try parallel.for_each_range(output.rows, 1024, (void*)&work, write_block)
    bytes.count += total

struct FixedWrite:
    var fields: const Field[]
    var stride: usize
    var rows: u8*
    var swap: bool

func write_block(context: void*, first: usize, end: usize, worker: usize):
    _ = worker
    let work = (FixedWrite*)context
    var offset: usize = 0
    for index in 0..<work.fields.length:
        let field = &work.fields[index]
        let target = work.rows + offset
        let row_bytes = work.stride
        offset += size_of(field.kind)
        if let source = field.source:
            if field.plain_float():
                # Bits are copied as they are, NaN payloads included.
                for row in first..<end:
                    store32(target + row * row_bytes, load32(source + row * field.stride, false), work.swap)
                continue
        for row in first..<end:
            store(target + row * row_bytes, field.kind, work.swap, value_at(field, row))

func write_rows(output: const Output*, swap: bool, bytes: Growing[u8]*) -> !:
    var scratch: u8[8]
    for row in 0..<output.rows:
        for at in 0..<output.fields.count:
            let field = &output.fields.view()[at]
            if not field.is_list:
                try put_value(bytes, scratch, field.kind, swap, value_at(field, row))
                continue
            let count = list_length(field, row)
            try put_value(bytes, scratch, field.count_kind, swap, (f64)count)
            for index in 0..<count: try put_value(bytes, scratch, field.kind, swap, list_value(field, row, index))

func put_value(bytes: Growing[u8]*, scratch: u8[], kind: Scalar, swap: bool, value: f64) -> !:
    store(scratch.data, kind, swap, value)
    let width = size_of(kind)
    for at in 0..<width: try bytes.push(scratch[at])

func write_ascii(output: const Output*, bytes: Growing[u8]*) -> !:
    var digits: u8[64]
    for row in 0..<output.rows:
        var first = true
        for at in 0..<output.fields.count:
            let field = &output.fields.view()[at]
            if not field.is_list:
                if not first: try bytes.push(' ')
                try put_number(bytes, digits, field.kind, value_at(field, row))
                first = false
                continue
            let count = list_length(field, row)
            if not first: try bytes.push(' ')
            try put_number(bytes, digits, field.count_kind, (f64)count)
            first = false
            for index in 0..<count:
                try bytes.push(' ')
                try put_number(bytes, digits, field.kind, list_value(field, row, index))
        try bytes.push(10)

## `value` as text for a property of `kind`: shortest round-trip digits for
## floats, the stored integer otherwise.
func put_number(bytes: Growing[u8]*, digits: u8[], kind: Scalar, value: f64) -> !:
    if kind == Scalar.float32:
        try put_text(bytes, try strings.format_f32((f32)value, digits))
        return
    if kind == Scalar.float64:
        try put_text(bytes, try strings.format_f64(value, digits))
        return
    var stored: u8[8]
    store(&stored[0], kind, false, value)
    try put_text(bytes, try strings.format_i64((i64)load(&stored[0], kind, false), digits))