Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-3d

Source of release 0.9.3

0.9.3 / src / renderers / gaussian_splats.lucb

## Gaussian splats in the viewport, and point clouds as dots. A
## GaussianSplats is packed once per cloud on the CPU (renderers/splat_data:
## centers, covariances, colors, SH, and merged levels of detail for large
## clouds) and uploads on its first draw; later draws (a camera drag) upload
## nothing. Each new view runs one compute pass that projects, culls and
## sorts the splats (renderers/splat_view), then draws one quad per visible
## splat back to front: shaders/splat.frag's Gaussian falloff, premultiplied,
## Blend.over, depth tested without writing depth, so meshes in front hide
## splats and splats veil meshes behind them. `draw` draws one cloud alone;
## a SplatScene draws several sorted together, so interleaved clouds blend
## in depth order. Selected splats (`set_marks`) are tinted.
from luce_gpu import gpu
import interop
import memory
from luce_std import math
import errors
from luce_geocore.geocore import Vector3, Matrix4, GeometrySet, Bounds
from luce_geocore.kernel import Selection, selection_bits
from cameras.perspective import PerspectiveCamera, Projection
from renderers.splat_data import SplatData, splat_data, splat_key, cloud_count
from renderers.splat_view import SplatView, SplatTimes, CloudInput, signature_words, open_view, close_view
from renderers.splat_shaders import splat_frag_words, splat_frag_msl, splat_frag_fast_math

## The flags of splat_params.glsl's counts.w.
let flag_srgb: u32 = 1
let flag_frames: u32 = 2
let flag_centers: u32 = 4
let flag_splats: u32 = 8
let flag_marks: u32 = 16
let flag_lod: u32 = 32
let flag_antialiased: u32 = 64

## How a cloud draws.
pub enum SplatMode as u8:
    ## Shaded Gaussians, their SH evaluated toward the eye.
    splats = 0
    ## A round dot at every center, its DC color (plain point clouds always).
    centers = 1

## How splats draw; the viewport's display options set it.
pub struct SplatLook:
    ## Splats or centers.
    pub var mode: SplatMode = SplatMode.splats
    ## The highest SH band evaluated, 0..3: lower is faster.
    pub var sh_degree: i64 = 3
    ## Splats less opaque than this are culled, and each one's quad ends
    ## where its Gaussian falls to it (3DGS's 1/255 by default).
    pub var alpha_cull: f64 = 0.00392156862745098
    ## Multiplies every splat's size (its standard deviations).
    pub var scale: f64 = 1.0
    ## The longest half axis a splat's quad takes, in pixels (navigation).
    pub var max_size: f64 = 2048.0
    ## A center dot's radius, in points.
    pub var dot_size: f64 = 1.5
    ## Level of detail: a merged run of splats whose footprint (3σ) is under
    ## this many pixels draws in place of its splats (0: never).
    pub var lod_pixels: f64 = 2.0
    ## ...once the splats drawn together number at least this many.
    pub var lod_from: i64 = 10000000

## A cloud's packed data on its device.
struct CloudBuffers:
    var device: gpu.Device
    var centers: gpu.Buffer
    var covariances: gpu.Buffer
    var colors: gpu.Buffer
    var frames: gpu.Buffer
    var harmonics: gpu.Buffer
    ## One bit per packed splat (u32 words): the selection to tint.
    var marks: gpu.Buffer

    func destroy():
        for buffer in [self.centers, self.covariances, self.colors, self.frames, self.harmonics, self.marks]:
            var owned = buffer
            owned.destroy()
        self.device.destroy()

## Serials tell clouds apart in a view's signature (an address can be reused).
var next_serial: u64 = 1

## One point cloud ready to draw: its packed data until the first draw
## uploads it, then its buffers.
pub struct GaussianSplats:
    var data: SplatData
    var settings: SplatLook
    var buffers: CloudBuffers?
    var uploads: i64
    let serial: u64
    ## The selection to tint, by packed splat (heap words; empty: none), its
    ## key, and whether the GPU copy is behind.
    var marks: u32[]
    var marks_key: u64
    var marks_changed: bool
    ## The view `draw` sorts in when it draws this cloud alone.
    var own_view: SplatView*?

    ## `geometry`'s point cloud placed by `matrix`: Gaussian splats when it
    ## has orient, scale and opacity, dots otherwise.
    pub static func of(geometry: interop.Reference[GeometrySet], matrix: Matrix4 = Matrix4()) -> interop.Reference[GaussianSplats]!:
        let made = try gaussian_splats(try geometry.get(), matrix)
        return try interop.Reference[GaussianSplats].adopt(made, gaussian_splats_type, memory.heap)

    ## How many points `geometry`'s cloud holds (0 without one).
    pub static func count(geometry: interop.Reference[GeometrySet]) -> i64!:
        return (i64)cloud_count(try geometry.get())

    ## What `of` would pack from (the cloud's columns, its color space and
    ## the placement): equal keys pack equal splats, so a held one is reused.
    pub static func key(geometry: interop.Reference[GeometrySet], matrix: Matrix4 = Matrix4()) -> i64!:
        return try splat_key(try geometry.get(), matrix)

    ## How it draws alone (a SplatScene sets its own look).
    pub func look() -> SplatLook:
        return self.settings

    ## Draw with `look` from now on: uploads nothing.
    pub func set_look(look: SplatLook) -> !:
        try check_look(look)
        self.settings = look

    ## Tint the points `selection` selects (a selection of the cloud's points,
    ## as an Edit node picks them; none clears). Equal selections upload once.
    pub func set_marks(selection: interop.Reference[Selection]?) -> !:
        if selection == none:
            if self.marks.length > 0:
                free(self.marks) in memory.heap
                var nothing: u32[]
                self.marks = nothing
                self.marks_key = 0
                self.marks_changed = true
            return
        let given = selection else return
        let chosen = try given.get()
        if chosen.key() == self.marks_key and self.marks.length > 0: return
        if chosen.domain() != 0 or (usize)chosen.size() != self.data.count: error(errors.invalid, "splat marks are a selection of the cloud's points")
        let bits = selection_bits(chosen)
        if self.marks.length == 0: self.marks = try new u32[(self.data.count + 31) // 32] --- in memory.heap
        for index in 0..<self.marks.length: self.marks[index] = 0
        let order = self.data.order
        for packed in 0..<self.data.count:
            let point = (usize)order[packed] if order.length > 0 else packed
            if bits.get(point): self.marks[packed >> 5] = self.marks[packed >> 5] | ((u32)1 << (u32)(packed & 31))
        self.marks_key = chosen.key()
        self.marks_changed = true

    ## The id of the positions it was packed from, as luce-geocore's
    ## `Mesh.positions_id` gives it: a mesh sharing the cloud's points (an
    ## Edit node's pick mesh) has the same id. Its selection tints this cloud
    ## and no other cloud of the same size.
    pub func source_id() -> i64:
        return (i64)(self.data.source >> 1)

    ## Whether a selection is tinted on it (`set_marks`).
    pub func has_marks() -> bool:
        return self.marks.length > 0

    ## Points it draws.
    pub func size() -> i64:
        return (i64)self.data.count

    ## Whether they are Gaussian splats (orient, scale, opacity), not plain points.
    pub func is_splats() -> bool:
        return self.data.splats

    ## Whether it has merged levels of detail (a splat cloud of at least 2^20).
    pub func has_lod() -> bool:
        return self.data.order.length > 0

    ## SH items per splat (0, 3, 8 or 15).
    pub func sh_items() -> i64:
        return (i64)self.data.items

    ## The world box of its centers (for framing).
    pub func bounds() -> Bounds:
        return Bounds(self.data.low, self.data.high)

    ## How many times its data went to the GPU (once, on its first draw).
    pub func upload_count() -> i64:
        return self.uploads

    ## How many views it sorted alone: a draw from an unchanged view sorts nothing.
    pub func sort_count() -> i64:
        let view = self.own_view else return 0
        return view.sort_count()

    ## Draw it alone over what `target` holds, through `camera`. A standalone
    ## frame (no device) draws nothing.
    pub func draw(camera: interop.Reference[PerspectiveCamera], target: interop.View[gpu.RenderTarget]) -> !:
        let native = try target.get()
        var device = gpu.device_of(native) catch failure:
            if failure.code != gpu.unavailable: error(failure.code, failure.message)
            recover gpu.Device()
        defer device.destroy()
        let size = try native.size()
        if not device.is_open() or size.pixel_width <= 0.0 or size.pixel_height <= 0.0: return
        if self.own_view == none: self.own_view = try open_view(device)
        let view = self.own_view else trap("splat view")
        var clouds: GaussianSplats*[1] = [&self]
        try draw_clouds(view, clouds, self.settings, camera, native, device, size.pixel_width, size.pixel_height, size.pixel_width / size.width)

    ## The GPU times of the last view it sorted alone (waiting for its pass).
    pub func timings() -> SplatTimes!:
        let view = self.own_view else error(errors.invalid, "no view was sorted yet")
        return try view.timings()

    func centers() -> bool:
        return self.settings.mode == SplatMode.centers or not self.data.splats

    ## The buffers on `device`: the packed data sent on the first draw (when
    ## the CPU copy is freed); the marks whenever they changed.
    func upload(device: gpu.Device) -> !:
        if self.buffers == none:
            var made = CloudBuffers(device = try device.share())
            errdefer made.destroy()
            made.centers = try gpu.Buffer.create(device, bytes_of_f32(self.data.centers))
            made.colors = try gpu.Buffer.create(device, bytes_of_f16(self.data.colors))
            if self.data.covariances.length > 0: made.covariances = try gpu.Buffer.create(device, bytes_of_f32(self.data.covariances))
            if self.data.frames.length > 0: made.frames = try gpu.Buffer.create(device, bytes_of_f16(self.data.frames))
            if self.data.harmonics.length > 0: made.harmonics = try gpu.Buffer.create(device, bytes_of_f16(self.data.harmonics))
            self.buffers = made
            self.uploads += 1
            self.data.release_arrays()
            self.marks_changed = true
        if not self.marks_changed: return
        var buffers = self.buffers else return
        if buffers.marks.is_open():
            buffers.marks.destroy()
        if self.marks.length > 0: buffers.marks = try gpu.Buffer.create(device, u8[]((u8*)self.marks.data, self.marks.length * 4))
        self.buffers = buffers
        self.marks_changed = false

    ## What a view needs of it for `projection` into a target of `width` by
    ## `height` pixels, `scale` pixels a point, drawn with `look`, LOD on or not.
    func input(projection: Projection, width: f64, height: f64, scale: f64, look: SplatLook, lod: bool) -> CloudInput:
        let buffers = self.buffers else trap("splats not uploaded")
        let eye = projection.position.subtract(self.data.origin)
        let forward = projection.backward.multiply_scalar(-1.0)
        let fx = projection.focal / projection.aspect * width * 0.5
        let fy = projection.focal * height * 0.5
        let centers = look.mode == SplatMode.centers or not self.data.splats
        var flags: u32 = 0
        if self.data.srgb: flags = flags | flag_srgb
        if buffers.frames.is_open(): flags = flags | flag_frames
        if centers: flags = flags | flag_centers
        if self.data.splats: flags = flags | flag_splats
        if buffers.marks.is_open(): flags = flags | flag_marks
        if self.has_lod(): flags = flags | flag_lod
        if self.data.antialiased: flags = flags | flag_antialiased
        let used = (look.sh_degree + 1) * (look.sh_degree + 1) - 1
        let rotation = self.data.rotation
        let uniforms: f32[32] = [(f32)eye.x, (f32)eye.y, (f32)eye.z, (f32)projection.near,
                                 (f32)projection.right.x, (f32)projection.right.y, (f32)projection.right.z, (f32)projection.far,
                                 (f32)projection.up.x, (f32)projection.up.y, (f32)projection.up.z, (f32)fx,
                                 (f32)forward.x, (f32)forward.y, (f32)forward.z, (f32)fy,
                                 (f32)width, (f32)height, 0.0, (f32)(look.lod_pixels if lod else 0.0),
                                 f32.bits((u32)self.data.count), f32.bits((u32)self.data.items), f32.bits((u32)used), f32.bits(flags),
                                 (f32)look.alpha_cull, (f32)look.scale, (f32)look.max_size, (f32)(look.dot_size * scale),
                                 (f32)rotation.x, (f32)rotation.y, (f32)rotation.z, (f32)rotation.w]
        return CloudInput(centers = buffers.centers, covariances = present_or(buffers.covariances, buffers.centers), colors = buffers.colors,
                          frames = present_or(buffers.frames, buffers.centers), harmonics = present_or(buffers.harmonics, buffers.centers),
                          marks = present_or(buffers.marks, buffers.centers), count = self.data.count, uniforms = uniforms)

    ## Free the packed data, the buffers and its own view.
    pub func close():
        if let view = self.own_view: close_view(view)
        self.own_view = none
        self.data.close()
        if self.marks.length > 0: free(self.marks) in memory.heap
        var nothing: u32[]
        self.marks = nothing
        if let buffers = self.buffers:
            var owned = buffers
            owned.destroy()
        self.buffers = none

## The interop type of a GaussianSplats, for adopting heap ones (Base callers).
pub let gaussian_splats_type: interop.Type[GaussianSplats] = interop.Type[GaussianSplats]("GaussianSplats", GaussianSplats.close, closeable = true, main_thread = true)

## A heap GaussianSplats of `set`'s point cloud placed by `matrix` (Base
## callers, such as a worker handing it to the UI thread: adopt it with
## `gaussian_splats_type` there). Fails when the set has no cloud.
pub func gaussian_splats(set: const GeometrySet*, matrix: Matrix4) -> GaussianSplats*!:
    var data = try splat_data(set, matrix)
    errdefer data.close()
    let serial = next_serial
    next_serial += 1
    return try new GaussianSplats(data = data, settings = SplatLook(), serial = serial) in memory.heap

## How many points `set`'s cloud holds (0 without one), for Base callers.
pub func splat_count(set: const GeometrySet*) -> i64:
    return (i64)cloud_count(set)

pub func check_look(look: SplatLook) -> !:
    if look.sh_degree < 0 or look.sh_degree > 3: error(errors.invalid, "a splat look's SH degree is 0..3")
    if not (look.alpha_cull >= 0.0001 and look.alpha_cull <= 0.99): error(errors.invalid, "a splat look's alpha cull is in 0.0001..0.99")
    if not (look.scale > 0.0 and look.scale <= 1000.0): error(errors.invalid, "a splat look's scale is in (0, 1000]")
    if not (look.max_size >= 1.0 and look.max_size <= 16384.0) or not (look.dot_size >= 0.25 and look.dot_size <= 64.0):
        error(errors.invalid, "a splat look needs a largest size of 1..16384 pixels and a dot size of 0.25..64 points")
    if not (look.lod_pixels >= 0.0 and look.lod_pixels <= 64.0) or look.lod_from < 0: error(errors.invalid, "a splat look's LOD size is 0..64 pixels from 0 splats on")

## Draw `clouds` sorted together in `view` with `look`: upload what is new,
## sort when the clouds or the camera changed, then draw. `target` is on
## `device`, `width` by `height` pixels, `scale` pixels a point.
pub func draw_clouds(view: SplatView*, clouds: GaussianSplats*[], look: SplatLook, camera: interop.Reference[PerspectiveCamera], target: const gpu.RenderTarget*,
                     device: gpu.Device, width: f64, height: f64, scale: f64) -> !:
    if not view.on(device): error(errors.invalid, "splats draw on the device their view was opened on")
    var total: i64 = 0
    for cloud in clouds:
        try cloud.upload(device)
        let buffers = cloud.buffers else continue
        var drawn = buffers.device
        if not drawn.same(&device): error(errors.invalid, "splats draw on the device they were first drawn on")
        total += (i64)cloud.data.count
    if total == 0: return
    let lod = look.lod_pixels > 0.0 and total >= look.lod_from
    let projection = try camera.value().projection(width / height)
    let inputs = try new CloudInput[clouds.length] --- in memory.heap
    defer free(inputs) in memory.heap
    let signature = try new u32[clouds.length * signature_words] --- in memory.heap
    defer free(signature) in memory.heap
    for index in 0..<clouds.length:
        let cloud = clouds[index]
        inputs[index] = cloud.input(projection, width, height, scale, look, lod)
        let words = signature[index * signature_words..<(index + 1) * signature_words]
        words[0] = (u32)(cloud.serial & 0xFFFFFFFF)
        words[1] = (u32)(cloud.serial >> 32)
        let uniforms = u32[]((u32*)&inputs[index].uniforms[0], 32)
        for at in 0..<(usize)32: words[2 + at] = uniforms[at]
        # A new selection resorts too (its tint is in the records).
        words[2 + 18] = (u32)(cloud.marks_key & 0xFFFFFFFF)
    if not view.sorted_for(signature): try view.sort_clouds(inputs, signature)
    let pipeline = try splat_pipeline(device, try target.pixel_format())
    try view.draw(target, pipeline, look.alpha_cull)

func present_or(buffer: gpu.Buffer, fallback: gpu.Buffer) -> gpu.Buffer:
    return buffer if buffer.is_open() else fallback

func bytes_of_f32(values: const f32[]) -> u8[]:
    return u8[]((u8*)values.data, values.length * 4)

## Halves as bytes, padded to whole words by the caller's even counts.
func bytes_of_f16(values: const f16[]) -> u8[]:
    return u8[]((u8*)values.data, values.length * 2)

# mark: The shared pipeline

## The splat shader and its pipelines on one device, per target format
## (slot 0 for presentation surfaces, 1 + the format otherwise): made on
## first use and kept. Main thread only, as drawing is.
struct SplatPipelines:
    var device: gpu.Device
    var shader: gpu.Shader
    var pipelines: (gpu.Pipeline?)[9]

var shared_pipelines: SplatPipelines? = none

## The splat pipeline drawing into targets of `pixel_format` on `device`:
## Blend.over, depth tested without writing depth.
func splat_pipeline(device: gpu.Device, pixel_format: gpu.Format?) -> gpu.Pipeline!:
    var wanted = device
    if let held = shared_pipelines:
        if not held.device.same(&wanted): release_splat_pipelines()
    if shared_pipelines == none:
        var shader = try gpu.Shader.create(device, splat_frag_words, splat_frag_msl, splat_frag_fast_math)
        errdefer shader.destroy()
        shared_pipelines = SplatPipelines(device = try device.share(), shader = shader)
    var held = shared_pipelines else trap("splat pipelines")
    var slot: usize = 0
    if let format = pixel_format: slot = (usize)(u8)format + 1
    if let made = held.pipelines[slot]: return made
    let made = try gpu.Pipeline.create(device, held.shader, gpu.Blend.over, pixel_format, depth_test = true)
    held.pipelines[slot] = made
    shared_pipelines = held
    return made

## Release the shared splat shader and pipelines (they are made again on the
## next draw): for a clean shutdown, or before closing the device.
pub func release_splat_pipelines():
    let held = shared_pipelines else return
    shared_pipelines = none
    for pipeline in held.pipelines:
        if let made = pipeline:
            var owned = made
            owned.destroy()
    var shader = held.shader
    shader.destroy()
    var device = held.device
    device.destroy()