dymokomi/luce-3d
Source of release 0.9.2
0.9.2 / src / renderers / gaussian_splats.lucb
## Gaussian splats in the viewport, and point clouds as dots. A
## GaussianSplats is packed once per cloud on the CPU (renderers/splat_data:
## centers, covariances, colors, SH, and merged levels of detail for large
## clouds) and uploads on its first draw; later draws (a camera drag) upload
## nothing. Each new view runs one compute pass that projects, culls and
## sorts the splats (renderers/splat_view), then draws one quad per visible
## splat back to front: shaders/splat.frag's Gaussian falloff, premultiplied,
## Blend.over, depth tested without writing depth, so meshes in front hide
## splats and splats veil meshes behind them. `draw` draws one cloud alone;
## a SplatScene draws several sorted together, so interleaved clouds blend
## in depth order. Selected splats (`set_marks`) are tinted.
from luce_gpu import gpu
import interop
import memory
from luce_std import math
import errors
from luce_geocore.geocore import Vector3, Matrix4, GeometrySet, Bounds
from luce_geocore.kernel import Selection, selection_bits
from cameras.perspective import PerspectiveCamera, Projection
from renderers.splat_data import SplatData, splat_data, splat_key, cloud_count
from renderers.splat_view import SplatView, SplatTimes, CloudInput, signature_words, open_view, close_view
from renderers.splat_shaders import splat_frag_words, splat_frag_msl, splat_frag_fast_math
## The flags of splat_params.glsl's counts.w.
let flag_srgb: u32 = 1
let flag_frames: u32 = 2
let flag_centers: u32 = 4
let flag_splats: u32 = 8
let flag_marks: u32 = 16
let flag_lod: u32 = 32
let flag_antialiased: u32 = 64
## How a cloud draws.
pub enum SplatMode as u8:
## Shaded Gaussians, their SH evaluated toward the eye.
splats = 0
## A round dot at every center, its DC color (plain point clouds always).
centers = 1
## How splats draw; the viewport's display options set it.
pub struct SplatLook:
## Splats or centers.
pub var mode: SplatMode = SplatMode.splats
## The highest SH band evaluated, 0..3: lower is faster.
pub var sh_degree: i64 = 3
## Splats less opaque than this are culled, and each one's quad ends
## where its Gaussian falls to it (3DGS's 1/255 by default).
pub var alpha_cull: f64 = 0.00392156862745098
## Multiplies every splat's size (its standard deviations).
pub var scale: f64 = 1.0
## The longest half axis a splat's quad takes, in pixels (navigation).
pub var max_size: f64 = 2048.0
## A center dot's radius, in points.
pub var dot_size: f64 = 1.5
## Level of detail: a merged run of splats whose footprint (3σ) is under
## this many pixels draws in place of its splats (0: never).
pub var lod_pixels: f64 = 2.0
## ...once the splats drawn together number at least this many.
pub var lod_from: i64 = 10000000
## A cloud's packed data on its device.
struct CloudBuffers:
var device: gpu.Device
var centers: gpu.Buffer
var covariances: gpu.Buffer
var colors: gpu.Buffer
var frames: gpu.Buffer
var harmonics: gpu.Buffer
## One bit per packed splat (u32 words): the selection to tint.
var marks: gpu.Buffer
func destroy():
for buffer in [self.centers, self.covariances, self.colors, self.frames, self.harmonics, self.marks]:
var owned = buffer
owned.destroy()
self.device.destroy()
## Serials tell clouds apart in a view's signature (an address can be reused).
var next_serial: u64 = 1
## One point cloud ready to draw: its packed data until the first draw
## uploads it, then its buffers.
pub struct GaussianSplats:
var data: SplatData
var settings: SplatLook
var buffers: CloudBuffers?
var uploads: i64
let serial: u64
## The selection to tint, by packed splat (heap words; empty: none), its
## key, and whether the GPU copy is behind.
var marks: u32[]
var marks_key: u64
var marks_changed: bool
## The view `draw` sorts in when it draws this cloud alone.
var own_view: SplatView*?
## `geometry`'s point cloud placed by `matrix`: Gaussian splats when it
## has orient, scale and opacity, dots otherwise.
pub static func of(geometry: interop.Reference[GeometrySet], matrix: Matrix4 = Matrix4()) -> interop.Reference[GaussianSplats]!:
let made = try gaussian_splats(try geometry.get(), matrix)
return try interop.Reference[GaussianSplats].adopt(made, gaussian_splats_type, memory.heap)
## How many points `geometry`'s cloud holds (0 without one).
pub static func count(geometry: interop.Reference[GeometrySet]) -> i64!:
return (i64)cloud_count(try geometry.get())
## What `of` would pack from (the cloud's columns, its color space and
## the placement): equal keys pack equal splats, so a held one is reused.
pub static func key(geometry: interop.Reference[GeometrySet], matrix: Matrix4 = Matrix4()) -> i64!:
return try splat_key(try geometry.get(), matrix)
## How it draws alone (a SplatScene sets its own look).
pub func look() -> SplatLook:
return self.settings
## Draw with `look` from now on: uploads nothing.
pub func set_look(look: SplatLook) -> !:
try check_look(look)
self.settings = look
## Tint the points `selection` selects (a selection of the cloud's points,
## as an Edit node picks them; none clears). Equal selections upload once.
pub func set_marks(selection: interop.Reference[Selection]?) -> !:
if selection == none:
if self.marks.length > 0:
free(self.marks) in memory.heap
var nothing: u32[]
self.marks = nothing
self.marks_key = 0
self.marks_changed = true
return
let given = selection else return
let chosen = try given.get()
if chosen.key() == self.marks_key and self.marks.length > 0: return
if chosen.domain() != 0 or (usize)chosen.size() != self.data.count: error(errors.invalid, "splat marks are a selection of the cloud's points")
let bits = selection_bits(chosen)
if self.marks.length == 0: self.marks = try new u32[(self.data.count + 31) // 32] --- in memory.heap
for index in 0..<self.marks.length: self.marks[index] = 0
let order = self.data.order
for packed in 0..<self.data.count:
let point = (usize)order[packed] if order.length > 0 else packed
if bits.get(point): self.marks[packed >> 5] = self.marks[packed >> 5] | ((u32)1 << (u32)(packed & 31))
self.marks_key = chosen.key()
self.marks_changed = true
## The id of the positions it was packed from, as luce-geocore's
## `Mesh.positions_id` gives it: a mesh sharing the cloud's points (an
## Edit node's pick mesh) has the same id. Its selection tints this cloud
## and no other cloud of the same size.
pub func source_id() -> i64:
return (i64)(self.data.source >> 1)
## Whether a selection is tinted on it (`set_marks`).
pub func has_marks() -> bool:
return self.marks.length > 0
## Points it draws.
pub func size() -> i64:
return (i64)self.data.count
## Whether they are Gaussian splats (orient, scale, opacity), not plain points.
pub func is_splats() -> bool:
return self.data.splats
## Whether it has merged levels of detail (a splat cloud of at least 2^20).
pub func has_lod() -> bool:
return self.data.order.length > 0
## SH items per splat (0, 3, 8 or 15).
pub func sh_items() -> i64:
return (i64)self.data.items
## The world box of its centers (for framing).
pub func bounds() -> Bounds:
return Bounds(self.data.low, self.data.high)
## How many times its data went to the GPU (once, on its first draw).
pub func upload_count() -> i64:
return self.uploads
## How many views it sorted alone: a draw from an unchanged view sorts nothing.
pub func sort_count() -> i64:
let view = self.own_view else return 0
return view.sort_count()
## Draw it alone over what `target` holds, through `camera`. A standalone
## frame (no device) draws nothing.
pub func draw(camera: interop.Reference[PerspectiveCamera], target: interop.View[gpu.RenderTarget]) -> !:
let native = try target.get()
var device = gpu.device_of(native) catch failure:
if failure.code != gpu.unavailable: error(failure.code, failure.message)
recover gpu.Device()
defer device.destroy()
let size = try native.size()
if not device.is_open() or size.pixel_width <= 0.0 or size.pixel_height <= 0.0: return
if self.own_view == none: self.own_view = try open_view(device)
let view = self.own_view else trap("splat view")
var clouds: GaussianSplats*[1] = [&self]
try draw_clouds(view, clouds, self.settings, camera, native, device, size.pixel_width, size.pixel_height, size.pixel_width / size.width)
## The GPU times of the last view it sorted alone (waiting for its pass).
pub func timings() -> SplatTimes!:
let view = self.own_view else error(errors.invalid, "no view was sorted yet")
return try view.timings()
func centers() -> bool:
return self.settings.mode == SplatMode.centers or not self.data.splats
## The buffers on `device`: the packed data sent on the first draw (when
## the CPU copy is freed); the marks whenever they changed.
func upload(device: gpu.Device) -> !:
if self.buffers == none:
var made = CloudBuffers(device = try device.share())
errdefer made.destroy()
made.centers = try gpu.Buffer.create(device, bytes_of_f32(self.data.centers))
made.colors = try gpu.Buffer.create(device, bytes_of_f16(self.data.colors))
if self.data.covariances.length > 0: made.covariances = try gpu.Buffer.create(device, bytes_of_f32(self.data.covariances))
if self.data.frames.length > 0: made.frames = try gpu.Buffer.create(device, bytes_of_f16(self.data.frames))
if self.data.harmonics.length > 0: made.harmonics = try gpu.Buffer.create(device, bytes_of_f16(self.data.harmonics))
self.buffers = made
self.uploads += 1
self.data.release_arrays()
self.marks_changed = true
if not self.marks_changed: return
var buffers = self.buffers else return
if buffers.marks.is_open():
buffers.marks.destroy()
if self.marks.length > 0: buffers.marks = try gpu.Buffer.create(device, u8[]((u8*)self.marks.data, self.marks.length * 4))
self.buffers = buffers
self.marks_changed = false
## What a view needs of it for `projection` into a target of `width` by
## `height` pixels, `scale` pixels a point, drawn with `look`, LOD on or not.
func input(projection: Projection, width: f64, height: f64, scale: f64, look: SplatLook, lod: bool) -> CloudInput:
let buffers = self.buffers else trap("splats not uploaded")
let eye = projection.position.subtract(self.data.origin)
let forward = projection.backward.multiply_scalar(-1.0)
let fx = projection.focal / projection.aspect * width * 0.5
let fy = projection.focal * height * 0.5
let centers = look.mode == SplatMode.centers or not self.data.splats
var flags: u32 = 0
if self.data.srgb: flags = flags | flag_srgb
if buffers.frames.is_open(): flags = flags | flag_frames
if centers: flags = flags | flag_centers
if self.data.splats: flags = flags | flag_splats
if buffers.marks.is_open(): flags = flags | flag_marks
if self.has_lod(): flags = flags | flag_lod
if self.data.antialiased: flags = flags | flag_antialiased
let used = (look.sh_degree + 1) * (look.sh_degree + 1) - 1
let rotation = self.data.rotation
let uniforms: f32[32] = [(f32)eye.x, (f32)eye.y, (f32)eye.z, (f32)projection.near,
(f32)projection.right.x, (f32)projection.right.y, (f32)projection.right.z, (f32)projection.far,
(f32)projection.up.x, (f32)projection.up.y, (f32)projection.up.z, (f32)fx,
(f32)forward.x, (f32)forward.y, (f32)forward.z, (f32)fy,
(f32)width, (f32)height, 0.0, (f32)(look.lod_pixels if lod else 0.0),
f32.bits((u32)self.data.count), f32.bits((u32)self.data.items), f32.bits((u32)used), f32.bits(flags),
(f32)look.alpha_cull, (f32)look.scale, (f32)look.max_size, (f32)(look.dot_size * scale),
(f32)rotation.x, (f32)rotation.y, (f32)rotation.z, (f32)rotation.w]
return CloudInput(centers = buffers.centers, covariances = present_or(buffers.covariances, buffers.centers), colors = buffers.colors,
frames = present_or(buffers.frames, buffers.centers), harmonics = present_or(buffers.harmonics, buffers.centers),
marks = present_or(buffers.marks, buffers.centers), count = self.data.count, uniforms = uniforms)
## Free the packed data, the buffers and its own view.
pub func close():
if let view = self.own_view: close_view(view)
self.own_view = none
self.data.close()
if self.marks.length > 0: free(self.marks) in memory.heap
var nothing: u32[]
self.marks = nothing
if let buffers = self.buffers:
var owned = buffers
owned.destroy()
self.buffers = none
## The interop type of a GaussianSplats, for adopting heap ones (Base callers).
pub let gaussian_splats_type: interop.Type[GaussianSplats] = interop.Type[GaussianSplats]("GaussianSplats", GaussianSplats.close, closeable = true, main_thread = true)
## A heap GaussianSplats of `set`'s point cloud placed by `matrix` (Base
## callers, such as a worker handing it to the UI thread: adopt it with
## `gaussian_splats_type` there). Fails when the set has no cloud.
pub func gaussian_splats(set: const GeometrySet*, matrix: Matrix4) -> GaussianSplats*!:
var data = try splat_data(set, matrix)
errdefer data.close()
let serial = next_serial
next_serial += 1
return try new GaussianSplats(data = data, settings = SplatLook(), serial = serial) in memory.heap
## How many points `set`'s cloud holds (0 without one), for Base callers.
pub func splat_count(set: const GeometrySet*) -> i64:
return (i64)cloud_count(set)
pub func check_look(look: SplatLook) -> !:
if look.sh_degree < 0 or look.sh_degree > 3: error(errors.invalid, "a splat look's SH degree is 0..3")
if not (look.alpha_cull >= 0.0001 and look.alpha_cull <= 0.99): error(errors.invalid, "a splat look's alpha cull is in 0.0001..0.99")
if not (look.scale > 0.0 and look.scale <= 1000.0): error(errors.invalid, "a splat look's scale is in (0, 1000]")
if not (look.max_size >= 1.0 and look.max_size <= 16384.0) or not (look.dot_size >= 0.25 and look.dot_size <= 64.0):
error(errors.invalid, "a splat look needs a largest size of 1..16384 pixels and a dot size of 0.25..64 points")
if not (look.lod_pixels >= 0.0 and look.lod_pixels <= 64.0) or look.lod_from < 0: error(errors.invalid, "a splat look's LOD size is 0..64 pixels from 0 splats on")
## Draw `clouds` sorted together in `view` with `look`: upload what is new,
## sort when the clouds or the camera changed, then draw. `target` is on
## `device`, `width` by `height` pixels, `scale` pixels a point.
pub func draw_clouds(view: SplatView*, clouds: GaussianSplats*[], look: SplatLook, camera: interop.Reference[PerspectiveCamera], target: const gpu.RenderTarget*,
device: gpu.Device, width: f64, height: f64, scale: f64) -> !:
if not view.on(device): error(errors.invalid, "splats draw on the device their view was opened on")
var total: i64 = 0
for cloud in clouds:
try cloud.upload(device)
let buffers = cloud.buffers else continue
var drawn = buffers.device
if not drawn.same(&device): error(errors.invalid, "splats draw on the device they were first drawn on")
total += (i64)cloud.data.count
if total == 0: return
let lod = look.lod_pixels > 0.0 and total >= look.lod_from
let projection = try camera.value().projection(width / height)
let inputs = try new CloudInput[clouds.length] --- in memory.heap
defer free(inputs) in memory.heap
let signature = try new u32[clouds.length * signature_words] --- in memory.heap
defer free(signature) in memory.heap
for index in 0..<clouds.length:
let cloud = clouds[index]
inputs[index] = cloud.input(projection, width, height, scale, look, lod)
let words = signature[index * signature_words..<(index + 1) * signature_words]
words[0] = (u32)(cloud.serial & 0xFFFFFFFF)
words[1] = (u32)(cloud.serial >> 32)
let uniforms = u32[]((u32*)&inputs[index].uniforms[0], 32)
for at in 0..<(usize)32: words[2 + at] = uniforms[at]
# A new selection resorts too (its tint is in the records).
words[2 + 18] = (u32)(cloud.marks_key & 0xFFFFFFFF)
if not view.sorted_for(signature): try view.sort_clouds(inputs, signature)
let pipeline = try splat_pipeline(device, try target.pixel_format())
try view.draw(target, pipeline, look.alpha_cull)
func present_or(buffer: gpu.Buffer, fallback: gpu.Buffer) -> gpu.Buffer:
return buffer if buffer.is_open() else fallback
func bytes_of_f32(values: const f32[]) -> u8[]:
return u8[]((u8*)values.data, values.length * 4)
## Halves as bytes, padded to whole words by the caller's even counts.
func bytes_of_f16(values: const f16[]) -> u8[]:
return u8[]((u8*)values.data, values.length * 2)
# mark: The shared pipeline
## The splat shader and its pipelines on one device, per target format
## (slot 0 for presentation surfaces, 1 + the format otherwise): made on
## first use and kept. Main thread only, as drawing is.
struct SplatPipelines:
var device: gpu.Device
var shader: gpu.Shader
var pipelines: (gpu.Pipeline?)[9]
var shared_pipelines: SplatPipelines? = none
## The splat pipeline drawing into targets of `pixel_format` on `device`:
## Blend.over, depth tested without writing depth.
func splat_pipeline(device: gpu.Device, pixel_format: gpu.Format?) -> gpu.Pipeline!:
var wanted = device
if let held = shared_pipelines:
if not held.device.same(&wanted): release_splat_pipelines()
if shared_pipelines == none:
var shader = try gpu.Shader.create(device, splat_frag_words, splat_frag_msl, splat_frag_fast_math)
errdefer shader.destroy()
shared_pipelines = SplatPipelines(device = try device.share(), shader = shader)
var held = shared_pipelines else trap("splat pipelines")
var slot: usize = 0
if let format = pixel_format: slot = (usize)(u8)format + 1
if let made = held.pipelines[slot]: return made
let made = try gpu.Pipeline.create(device, held.shader, gpu.Blend.over, pixel_format, depth_test = true)
held.pipelines[slot] = made
shared_pipelines = held
return made
## Release the shared splat shader and pipelines (they are made again on the
## next draw): for a clean shutdown, or before closing the device.
pub func release_splat_pipelines():
let held = shared_pipelines else return
shared_pipelines = none
for pipeline in held.pipelines:
if let made = pipeline:
var owned = made
owned.destroy()
var shader = held.shader
shader.destroy()
var device = held.device
device.destroy()