Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-3d

Source of release 0.9.6

0.9.6 / src / renderers / splat_data.lucb

## A point cloud's static GPU data for GaussianSplats, packed on the CPU once
## per cook result (on a worker, off the UI thread) and uploaded on the first
## draw. A splat cloud (luce-geocore's conventions: orient, scale, opacity,
## linear Cd, the `sh` array, optional restorient) packs per splat:
## - its center, placed, relative to the pack origin (f32 × 3), and its opacity;
## - its placed covariance L Σ Lᵀ (f32 × 6: xx, xy, xz, yy, yz, zz);
## - its DC color in the cloud's encoding, encode(Cd) (f16 × 4), to which the
##   shader adds the SH bands before decoding, as display_color does;
## - its SH frame orient · restorient⁻¹ (f16 × 4), when it has a restorient;
## - its SH items (f16, RGB per item, padded to whole u32 words).
## A plain cloud packs centers and Cd only, drawn as dots. A splat cloud of
## at least `lod_minimum` splats is then put in Morton order with merged
## twigs and chunks after it (renderers/splat_lod), its `order` mapping
## packed positions back to points.
from luce_std import math, parallel
import memory
import errors
from luce_geocore.geocore import Vector3, Matrix4, GeometrySet
from luce_geocore.kernel import AttributeStore, Attribute, Domain, DataType, set_cloud, cloud_store, cloud_positions, cloud_origin, cloud_positions_id, set_detail
from luce_geocore.splats import Quat, Mat3, Floats, point_floats, is_splat_store, sh_attribute, sh_items, detail_color_space, ColorSpace, from_linear
from luce_geocore.splats import compose_covariance, rotation_part, sh_frame, restorient_name, orient_name, scale_name, opacity_name, color_name
from renderers.splat_lod import LodArrays, build_lod, lod_minimum

## The most points one cloud draws: 2^26.
pub let splat_limit: usize = 67108864
## A plain cloud's color without Cd: light gray.
let plain_gray: f64 = 0.8

## The packed arrays (heap, owned) and what the shader needs to read them.
pub struct SplatData:
    ## Per point: x, y, z from `origin`, and the opacity (1 for plain points).
    pub var centers: f32[]
    ## Per splat: the covariance's xx, xy, xz, yy, yz, zz (empty for plain points).
    pub var covariances: f32[]
    ## Per point: the DC color (encode(Cd) for splats, linear Cd for points) and 0.
    pub var colors: f16[]
    ## Per splat: its SH frame quaternion x, y, z, w (empty without restorient).
    pub var frames: f16[]
    ## Per splat: `stride` halves, its SH items (RGB each) then zeros.
    pub var harmonics: f16[]
    ## With level of detail: the point at each packed position (empty: the
    ## points' own order). Centers, covariances and colors then also hold the
    ## merged twigs and chunks after the `count` splats.
    pub var order: u32[]
    ## Points packed.
    pub let count: usize
    ## SH items per splat (0, 3, 8 or 15).
    pub let items: usize
    ## Halves stored per splat in `harmonics` (items × 3, rounded up to even).
    pub let stride: usize
    ## The world point the centers are relative to.
    pub let origin: Vector3
    ## The placement's rotation, turning SH-frame (and cloud) directions to world ones.
    pub let rotation: Quat
    ## Colors and SH are sRGB-encoded (decoded after the bands are added).
    pub let srgb: bool
    ## Gaussian splats (orient, scale, opacity) rather than plain points.
    pub let splats: bool
    ## The packed cloud's positions column id (luce-geocore's change ids):
    ## which cloud it is, whatever else has as many points.
    pub let source: u64
    ## Trained with Mip-Splatting's 2D filter (detail `gsplat_antialiased`):
    ## the projection compensates the low-pass's opacity.
    pub let antialiased: bool
    ## The placed centers' box: its low corner.
    pub var low: Vector3
    ## Its high corner.
    pub var high: Vector3

    ## Free everything.
    pub func close():
        self.release_arrays()
        if self.order.length > 0: free(self.order) in memory.heap
        var indices: u32[]
        self.order = indices

    ## Free the packed arrays once uploaded (the order stays: selections map
    ## through it).
    pub func release_arrays():
        if self.centers.length > 0: free(self.centers) in memory.heap
        if self.covariances.length > 0: free(self.covariances) in memory.heap
        if self.colors.length > 0: free(self.colors) in memory.heap
        if self.frames.length > 0: free(self.frames) in memory.heap
        if self.harmonics.length > 0: free(self.harmonics) in memory.heap
        var floats: f32[]
        var halves: f16[]
        self.centers = floats
        self.covariances = floats
        self.colors = halves
        self.frames = halves
        self.harmonics = halves

## The points `set`'s cloud holds (0 without one).
pub func cloud_count(set: const GeometrySet*) -> usize:
    let cloud = set_cloud(set) else return 0
    return cloud.count()

## What `splat_data` packs from (the cloud's columns, color space and
## `matrix`) as a key: equal keys pack equal data.
pub func splat_key(set: const GeometrySet*, matrix: Matrix4) -> i64!:
    let cloud = set_cloud(set) else error(errors.invalid, "the geometry has no point cloud")
    let store = cloud_store(cloud)
    var key = mix_id(0xCBF29CE484222325, cloud_positions_id(cloud))
    for name in [orient_name, scale_name, opacity_name, color_name, "sh", restorient_name]:
        let index = store.find(name, Domain.point)
        if index < 0:
            key = mix_id(key, 0)
            continue
        let entry = store.entry((usize)index)
        key = mix_id(mix_id(mix_id(key, entry.data.id()), entry.offsets.id()), (u64)(entry.kind) + (u64)entry.width * 16)
    key = mix_id(key, (u64)(u8)detail_color_space(set_detail(set)))
    key = mix_id(key, 1 if is_antialiased(set) else 0)
    let origin = cloud_origin(cloud)
    for value in [matrix.x, matrix.y, matrix.z, matrix.translation, origin]:
        key = mix_value(mix_value(mix_value(key, value.x), value.y), value.z)
    return (i64)(key >> 1)

## Whether `set` records an antialiased capture: detail `gsplat_antialiased`
## nonzero (luce-spz reads it from an SPZ's flags).
pub func is_antialiased(set: const GeometrySet*) -> bool:
    let detail = set_detail(set)
    let index = detail.find(antialiased_name, Domain.detail)
    if index < 0: return false
    let entry = detail.entry((usize)index)
    return entry.kind != DataType.text and entry.number(0, 0) != 0.0

## The detail luce-spz records an SPZ's antialiased flag in.
pub let antialiased_name: str = "gsplat_antialiased"

func mix_id(key: u64, id: u64) -> u64:
    return (key ^ id) *% 0x100000001B3

func mix_value(key: u64, value: f64) -> u64:
    var bits = value
    return mix_id(key, memory.read[u64]((void*)&bits))

## The inputs and outputs of one packing, shared by its workers.
struct Packing:
    let positions: const f32[]
    let linear: Mat3
    let identity: bool
    let orient: Floats
    let scale: Floats
    let opacity: Floats
    let color: Floats
    let restorient: Floats
    let harmonics_source: const Attribute*?
    let items: usize
    let stride: usize
    let space: ColorSpace
    let splats: bool
    var data: SplatData*

## Pack `set`'s point cloud placed by `matrix` (see the module comment).
pub func splat_data(set: const GeometrySet*, matrix: Matrix4) -> SplatData!:
    let cloud = set_cloud(set) else error(errors.invalid, "the geometry has no point cloud")
    let count = cloud.count()
    if count == 0: error(errors.invalid, "the point cloud is empty")
    if count > splat_limit: error(errors.invalid, "a point cloud draws at most 2^26 points")
    if not matrix.is_finite(): error(errors.invalid, "a splat placement must be finite")
    let store = cloud_store(cloud)
    let splats = is_splat_store(store)
    let space = detail_color_space(set_detail(set))
    let linear = Mat3.of_matrix4(matrix)
    var rotation = Quat.identity()
    if linear.determinant() > 0.0: rotation = Quat.of_matrix(rotation_part(linear))
    let origin = matrix.transform_point(cloud_origin(cloud))
    let positions = cloud_positions(cloud)
    let items = sh_items(store) if splats else 0
    let stride = (items * 3 + 1) // 2 * 2
    var restorient = Floats()
    if splats:
        if let found = point_floats(store, restorient_name, 4): restorient = found
    var harmonics_source: const Attribute*? = none
    let sh_index = sh_attribute(store)
    if splats and sh_index >= 0: harmonics_source = store.entry((usize)sh_index)
    var data = SplatData(count = count, items = items, stride = stride, origin = origin, rotation = rotation, srgb = splats and space == ColorSpace.srgb,
                         splats = splats, source = cloud_positions_id(cloud), antialiased = splats and is_antialiased(set), low = Vector3(), high = Vector3())
    errdefer data.close()
    data.centers = try new f32[count * 4] --- in memory.heap
    data.colors = try new f16[count * 4] --- in memory.heap
    if splats:
        data.covariances = try new f32[count * 6] --- in memory.heap
        if restorient.present(): data.frames = try new f16[count * 4] --- in memory.heap
        if stride > 0: data.harmonics = try new f16[count * stride] --- in memory.heap
    var work = Packing(positions = f32[]((f32*)positions.data, count * 3), linear = linear, identity = matrix.is_identity(),
                       orient = floats_of(store, orient_name, 4), scale = floats_of(store, scale_name, 3), opacity = floats_of(store, opacity_name, 1),
                       color = floats_of(store, color_name, 3), restorient = restorient, harmonics_source = harmonics_source, items = items, stride = stride,
                       space = space, splats = splats, data = &data)
    try parallel.for_each_range(count, 16384, (void*)&work, pack_range)
    # The box of the placed centers.
    var low = Vector3(1.0e300, 1.0e300, 1.0e300)
    var high = Vector3(-1.0e300, -1.0e300, -1.0e300)
    for index in 0..<count:
        let x = (f64)data.centers[index * 4]
        let y = (f64)data.centers[index * 4 + 1]
        let z = (f64)data.centers[index * 4 + 2]
        low = Vector3(math.min(low.x, x), math.min(low.y, y), math.min(low.z, z))
        high = Vector3(math.max(high.x, x), math.max(high.y, y), math.max(high.z, z))
    data.low = origin.add(low)
    data.high = origin.add(high)
    if splats and count >= lod_minimum:
        var arrays = LodArrays(centers = data.centers, covariances = data.covariances, colors = data.colors, frames = data.frames,
                               harmonics = data.harmonics, stride = stride, space = space)
        let order = try build_lod(&arrays, count, low, high)
        data.centers = arrays.centers
        data.covariances = arrays.covariances
        data.colors = arrays.colors
        data.frames = arrays.frames
        data.harmonics = arrays.harmonics
        data.order = order
    return data

func floats_of(store: const AttributeStore*, name: str, width: usize) -> Floats:
    return point_floats(store, name, width) else Floats()

## Pack points first..<end.
func pack_range(context: void*, first: usize, end: usize, worker: usize) -> !:
    _ = worker
    let work = (Packing*)context
    let data = work.data
    for index in first..<end:
        let local = Vector3((f64)work.positions[index * 3], (f64)work.positions[index * 3 + 1], (f64)work.positions[index * 3 + 2])
        let placed = local if work.identity else work.linear.transform(local)
        data.centers[index * 4] = (f32)placed.x
        data.centers[index * 4 + 1] = (f32)placed.y
        data.centers[index * 4 + 2] = (f32)placed.z
        data.centers[index * 4 + 3] = (f32)math.clamp(work.opacity.at(index, 0), 0.0, 1.0) if work.splats else 1.0
        # The DC color in the cloud's encoding (plain clouds: linear Cd).
        for channel in 0..<(usize)3:
            var value = plain_gray if not work.splats else 0.5
            if work.color.present(): value = work.color.at(index, channel)
            if work.splats: value = from_linear(value, work.space)
            data.colors[index * 4 + channel] = (f16)math.clamp(value, -60000.0, 60000.0)
        data.colors[index * 4 + 3] = (f16)0.0
        if not work.splats: continue
        let orient = Quat(work.orient.at(index, 0), work.orient.at(index, 1), work.orient.at(index, 2), work.orient.at(index, 3))
        let turn = orient.normalized() if orient.x * orient.x + orient.y * orient.y + orient.z * orient.z + orient.w * orient.w > 1.0e-30 else Quat.identity()
        let scale = Vector3(math.abs(work.scale.at(index, 0)), math.abs(work.scale.at(index, 1)), math.abs(work.scale.at(index, 2)))
        var sigma = compose_covariance(turn, scale)
        if not work.identity: sigma = work.linear.multiply(sigma).multiply(work.linear.transposed())
        let m = sigma.m
        let packed: f64[6] = [m[0], m[1], m[2], m[4], m[5], m[8]]
        for at in 0..<(usize)6: data.covariances[index * 6 + at] = (f32)packed[at]
        if data.frames.length > 0:
            let rest = Quat(work.restorient.at(index, 0), work.restorient.at(index, 1), work.restorient.at(index, 2), work.restorient.at(index, 3))
            let rest_unit = rest.normalized() if rest.x * rest.x + rest.y * rest.y + rest.z * rest.z + rest.w * rest.w > 1.0e-30 else Quat.identity()
            let frame = sh_frame(turn, rest_unit)
            let values: f64[4] = [frame.x, frame.y, frame.z, frame.w]
            for at in 0..<(usize)4: data.frames[index * 4 + at] = (f16)values[at]
        if work.stride > 0:
            let out = data.harmonics[index * work.stride..<(index + 1) * work.stride]
            for at in 0..<out.length: out[at] = (f16)0.0
            let source = work.harmonics_source else continue
            var have = source.item_count(index)
            if have > work.items: have = work.items
            let values = f32[]((f32*)source.data.view().data, source.data.view().length // 4)
            let start = source.first_item(index) * 3
            for at in 0..<have * 3: out[at] = (f16)math.clamp((f64)values[start + at], -60000.0, 60000.0)