dymokomi/luce-geocore
Source of release 0.7.2
0.7.2 / src / code / cook.lucb
# The cook: compile (or find) the program, bind the columns, run, and build the output
# set from what the program wrote.
## The timeline and the spare parameters' values for one cook.
struct Settings:
let names: const str[]
let values: const f64[]
let frame: f64
let time: f64
let fps: f64
## Run the snippet over `set` on `threads` workers (0: every core): the new set, or an
## error naming the line, column and element (see `fault_message`). Run Over Voxels runs
## over grid `volume` ("" the first), made dense over its leaves' box first when `dense`.
## Point clouds run as meshes of their points (geometry_set.cloud_mesh): Run Over Points
## on a set without a mesh runs over its cloud and gives a cloud back (and a mesh of the
## polygons the snippet added, over copies of their points), and an input's cloud is what `k.input(i)` reads of it when
## the input has no mesh.
pub func run_code(set: const GeometrySet*, input1: const GeometrySet*?, run_over: i64, header: str, body: str, group: str, group_type: i64, names: const str[], values: const f64[], frame: f64, time: f64, fps: f64, count: i64, threads: usize, report: CookReport*? = none, volume: str = "", dense: bool = false) -> GeometrySet*!:
var first: GeometrySet*? = none
defer:
if let made = first: dispose_set(made)
var second: GeometrySet*? = none
defer:
if let made = second: dispose_set(made)
if run_over == 1: first = try points_as_mesh(set)
if let other = input1: second = try points_as_mesh(other)
var source: const GeometrySet* = set
if let made = first: source = made
var other_source: const GeometrySet*? = input1
if let made = second: other_source = made
let result = try run_sets(source, other_source, run_over, header, body, group, group_type, names, values, frame, time, fps, count, threads, report, volume, dense)
if first == none: return result
let cloud = set_cloud(set) else return result
mesh_as_points(result, cloud) catch failure:
dispose_set(result)
error(failure.code, failure.message)
return result
## A copy of `set` with its point cloud as a mesh of its points, when it has a cloud and
## no mesh; else none.
func points_as_mesh(set: const GeometrySet*) -> GeometrySet*?!:
if set_mesh(set) != none: return none
let cloud = set_cloud(set) else return none
let storage = try set.storage()
errdefer dispose_set(storage)
try storage.put(&mesh_component, (void*)(try cloud_mesh(cloud)))
storage.remove(&points_component)
return storage
## Turn a run over a cloud back into a cloud, keeping `like`'s detail attributes: every
## point the run kept or added, with what it wrote, stays a point of the cloud (splats
## stay splats). Polygons the snippet added become a mesh beside it, made of copies of
## the points they use, as Houdini's polygons over a splat cloud's points would be.
func mesh_as_points(storage: GeometrySet*, like: const PointCloud*) -> !:
let mesh = set_mesh(storage) else return
try storage.put(&points_component, (void*)(try cloud_of_mesh(mesh, like)))
if mesh.face_count() == 0:
storage.remove(&mesh_component)
return
try storage.put(&mesh_component, (void*)(try compacted_storage(mesh)))
func run_sets(set: const GeometrySet*, input1: const GeometrySet*?, run_over: i64, header: str, body: str, group: str, group_type: i64, names: const str[], values: const f64[], frame: f64, time: f64, fps: f64, count: i64, threads: usize, report: CookReport*?, volume: str, dense: bool) -> GeometrySet*!:
let mode = try RunOver.of(run_over)
_ = try groups.GroupType.of(group_type)
if names.length != values.length:
error(errors.invalid, "every parameter name needs a value")
if count < 0:
error(errors.invalid, "Numbers runs a count of zero or more")
let mesh = set_mesh(set)
var lanes: usize = 1
var run_room: VoxelRun[1] = ---
var run_pointer: VoxelRun*? = none
defer release_run(run_pointer)
if mode == .numbers:
lanes = (usize)count
elif mode == .voxels:
let made = try voxel_run(set, volume, dense) else return try set.storage()
run_room[0] = made
run_pointer = &run_room[0]
lanes = made.target.leaf_count() * leaf_voxels
elif mode != .detail:
let source = mesh else return try set.storage()
lanes = try source.domain_size(mode.domain())
var layout = try build_layout(mode, mesh, set, run_pointer)
defer layout.close()
let entry = try acquire(mode, header, body, &layout)
defer release(entry)
let compiled = &entry.compiled
if not compiled.ok():
var room: u8[1024]
error(errors.invalid, check_failure(room, compiled, header, body))
if lanes == 0:
return try set.storage()
let settings = Settings(names = names, values = values, frame = frame, time = time, fps = fps)
var cook = Cook(mode = mode, mesh = mesh, set = set, lanes = lanes, program = &compiled.program, voxels = run_pointer, dense = dense)
defer cook.close()
# What the snippet's k.input() reads and the edits it queues.
var state: CookState
defer state.close()
state.first.mesh = mesh
state.first.set = set
if let other = input1:
state.second.mesh = set_mesh(other)
state.second.set = other
var old_points: usize = 0
var old_prims: usize = 0
if let source = mesh:
old_points = (usize)source.point_count()
old_prims = (usize)source.face_count()
try state.edits.prepare(lanes, old_points, old_prims)
prepare_sources(&state, &compiled.program)
state.ramps = try gather_ramps(compiled, names, values, header, body)
try prepare_named(&state, compiled, header, body)
try prepare_sh(&state, &compiled.program, mode)
try cook.bind_inputs(&layout)
try cook.bind_outputs(&layout)
try cook.bind_uniforms(&settings)
if let run = run_pointer:
cook.active = try voxel_active(run, &cook.scratch)
elif not groups.is_blank(group) and mode != .detail and mode != .numbers:
try cook.restrict(group, group_type)
let request = Request(count = lanes, inputs = cook.inputs.view(), outputs = cook.engine_outputs.view(), uniforms = cook.uniforms.view(), threads = threads, active = cook.active, context = (void*)&state)
let workers = engine.workers_of(&request)
try state.edits.prepare_sets(workers)
try state.clouds.prepare(workers)
try state.log.prepare(workers)
try state.values.prepare(workers)
state.program = &compiled.program
if let fault = try engine.run(&compiled.program, &request):
var room: u8[1024]
error(errors.invalid, fault_message(room, compiled, fault, mode, header, body))
if let told = report:
try told.take(&state.log)
let storage = try cook.output()
put_sh(&state, storage) catch failure:
dispose_set(storage)
error(failure.code, failure.message)
if not state.edits.any():
return storage
errdefer dispose_set(storage)
let made = try applied(set_mesh(storage), &state.edits)
try storage.put(&mesh_component, (void*)made)
return storage
## What a cook's snippet told the node: its printf output (the first messages in element
## order and a line counting the rest), how many messages there were, and its first
## warning with how many there were.
pub struct CookReport:
pub var output: collections.List[u8]
pub var printed: u64
pub var warning: collections.List[u8]
pub var warnings: u64
pub func close():
self.output.destroy()
self.warning.destroy()
## Copy what `log` holds.
func take(log: const Log*) -> !:
self.printed = try log.output(&self.output)
let (first, count) = log.warning()
self.warnings = count
for byte in first.bytes: try self.warning.append(byte)
## One cook's columns and buffers.
struct Cook:
let mode: RunOver
let mesh: const Mesh*?
let set: const GeometrySet*
let lanes: usize
let program: const Program*
## Run Over Voxels' target, and whether it was made dense.
let voxels: const VoxelRun*?
let dense: bool
var scratch: Scratch
var inputs: collections.List[Column]
var engine_outputs: collections.List[Column]
var outputs: collections.List[Output]
var uniforms: collections.List[Uniform]
var active: const u8[]
var positions: SharedColumn[Float3]
var has_positions: bool
func close():
for at in 0..<self.outputs.length():
var output = self.outputs.at(at)
if output.has_attribute: output.attribute.release()
if self.has_positions: self.positions.release()
self.scratch.close()
self.inputs.destroy()
self.engine_outputs.destroy()
self.outputs.destroy()
self.uniforms.destroy()
## An engine column for every column the program reads.
func bind_inputs(layout: const Layout*) -> !:
for binding in self.program.inputs():
if self.has_input(binding.name):
continue
let column = layout.find(binding.name) else error(errors.invalid, "the program reads a column the input lacks")
if let run = self.voxels:
try self.inputs.append(try voxel_input(column, run, &self.scratch))
else:
try self.inputs.append(try input_of(column, self.mesh, self.set, &self.scratch))
func has_input(name: str) -> bool:
for column in self.inputs.view():
if column.name == name: return true
return false
## A buffer for every column the program writes, holding the input's values first.
func bind_outputs(layout: const Layout*) -> !:
for binding in self.program.outputs():
var known = false
for output in self.outputs.view():
if output.name == binding.name: known = true
if known:
continue
var width: usize = 1
for other in self.program.outputs():
if other.name == binding.name and (usize)other.component + 1 > width:
width = (usize)other.component + 1
let existing = layout.find(binding.name)
if let column = existing:
if column.components > width and column.kind == binding.kind: width = column.components
try self.add_output(binding.name, binding.kind, width, existing)
func add_output(name: str, kind: Kind, width: usize, existing: LayoutColumn?) -> !:
var none_yet: u8[]
var output = Output(name = name, kind = kind, width = width, bytes = none_yet, attribute = Attribute(), has_attribute = false)
if let run = self.voxels:
# Voxel outputs: the target's values and grids, and what is not kept (`P`).
if not strings.starts_with(name, "detail:"):
output.bytes = try self.scratch.bytes(self.lanes * width * kind.size())
if kind == .f32: try prefill_voxels(run, &output, existing)
try self.outputs.append(output)
try self.engine_outputs.append(output_column(name, kind, output.bytes, width))
return
if name == "P":
let source = self.mesh else error(errors.invalid, "positions without a mesh")
self.positions = try SharedColumn[Float3].copy_of(source.point_span())
self.has_positions = true
let span = self.positions.span()
output.bytes = u8[]((u8*)span.data, span.length * 12)
let origin = source.position_origin()
if origin.x != 0.0 or origin.y != 0.0 or origin.z != 0.0:
output.bytes = try self.scratch.bytes(span.length * 12)
let world = f32_view(output.bytes)
for at in 0..<span.length:
world[at * 3] = (f32)((f64)span[at].x + origin.x)
world[at * 3 + 1] = (f32)((f64)span[at].y + origin.y)
world[at * 3 + 2] = (f32)((f64)span[at].z + origin.z)
elif strings.starts_with(name, "group:") or strings.starts_with(name, "detail:"):
output.bytes = try self.scratch.bytes(self.lanes * width * kind.size())
if let column = existing:
try self.prefill(&output, column)
else:
let data_type = data_type_of(kind) else error(errors.invalid, "a column type no attribute stores")
output.attribute = try Attribute.create(name, self.mode.domain(), data_type, width, self.lanes)
output.has_attribute = true
output.bytes = output.attribute.data.span()
# Elements the run skips (outside its group) keep 0 in a new attribute.
memory.set(output.bytes, 0)
if let column = existing:
if column.origin == .attribute or column.origin == .narrowed:
let source = self.mesh else return
fill_from(output.bytes, kind, width, self.lanes, try source.attribute_entry((i64)column.index))
try self.outputs.append(output)
try self.engine_outputs.append(output_column(name, kind, output.bytes, width))
## Start a group or detail output with the input's values.
func prefill(output: Output*, column: LayoutColumn) -> !:
if column.origin == .group:
let source = self.mesh else return
let entry = try source.attribute_entry((i64)column.index)
let members = Bits.of_bytes(entry.data.view(), entry.elements)
for at in 0..<self.lanes: output.bytes[at] = (u8)members.get(at)
elif column.origin == .detail:
let entry = set_detail(self.set).entry(column.index)
let overlap = output.width if output.width < entry.width else entry.width
for lane in 0..<self.lanes:
for k in 0..<overlap:
put_number(output.bytes, output.kind, lane * output.width + k, entry.number(0, k))
## Every uniform's value: the timeline, the counts, detail values and parameters.
func bind_uniforms(settings: const Settings*) -> !:
for slot in self.program.uniforms():
let value = self.uniform_number(slot, settings)
try self.uniforms.append(Uniform(name = slot.name, kind = slot.kind, bits = number_bits(slot.kind, value, slot.default)))
func uniform_number(slot: engine.UniformSlot, settings: const Settings*) -> f64?:
let name = slot.name
if name == "frame": return settings.frame
if name == "time": return settings.time
if name == "fps": return settings.fps
if name == "numpt" or name == "numprim" or name == "numvtx": return (f64)self.lanes
if strings.starts_with(name, "res["):
let run = self.voxels else return 0.0
let (_, component) = split_component(name)
return run.res[component] if component < 3 else 0.0
if strings.starts_with(name, "detail:"):
let (base, component) = split_component((str)name.bytes[7..])
let store = set_detail(self.set)
let index = store.find(base, Domain.detail)
if index < 0: return 0.0
let entry = store.entry((usize)index)
return entry.number(0, component) if component < entry.width else 0.0
for (at, given) in settings.names.indexed():
if given == name: return settings.values[at]
return none
## Run only on the elements of group expression `group`, read on `group_type`'s
## domain (0 guesses it from the expression) and converted to the run's domain as
## verbs convert theirs.
func restrict(group: str, group_type: i64) -> !:
let source = self.mesh else return
let target = self.mode.domain()
let named = try groups.GroupType.of(group_type)
var domain = named.domain(target)
if named == .guess: domain = try groups.guess_domain(source, group, target)
var result = try groups.group_bits(source, group, domain)
defer result.bits.release()
var chosen = try BitsBuilder.copy_of(Bits(result.bits.view(), try source.domain_size(domain)))
defer chosen.close()
if domain != target:
let topology = try Topology.of(source)
var moved = try groups.converted(&topology, chosen.view(), domain, target, groups.strict_conversion(domain, target))
chosen.close()
chosen = moved
let mask = try self.scratch.bytes(self.lanes)
let members = chosen.view()
for at in 0..<self.lanes: mask[at] = (u8)members.get(at)
self.active = mask
## The output set: the mesh with new positions and attributes, new detail values.
func output() -> GeometrySet*!:
if let run = self.voxels:
let storage = try voxel_output(run, self.set, self.outputs.view(), self.dense)
errdefer dispose_set(storage)
for output in self.outputs.view():
if strings.starts_with(output.name, "detail:"):
try set_put_detail(storage, try self.detail_attribute(output))
return storage
let storage = try self.set.storage()
errdefer dispose_set(storage)
var mesh_changed = self.has_positions
for output in self.outputs.view():
if not strings.starts_with(output.name, "detail:"): mesh_changed = true
if mesh_changed:
let source = self.mesh else return storage
let made = try self.output_mesh(source)
try storage.put(&mesh_component, (void*)made)
for output in self.outputs.view():
if strings.starts_with(output.name, "detail:"):
try set_put_detail(storage, try self.detail_attribute(output))
return storage
func output_mesh(source: const Mesh*) -> Mesh*!:
let made = try self.moved_mesh(source)
errdefer mesh_dispose(made)
for at in 0..<self.outputs.length():
var output = self.outputs.at(at)
if output.has_attribute:
output.has_attribute = false
self.outputs.set(at, output)
try put_attribute(made, output.attribute)
elif strings.starts_with(output.name, "group:"):
try put_attribute(made, try self.group_attribute(output, source))
return made
## The mesh with the written positions (or sharing the old ones).
func moved_mesh(source: const Mesh*) -> Mesh*!:
if not self.has_positions:
return try mesh_share(source)
let points = self.positions.span()
let origin = source.position_origin()
if origin.x != 0.0 or origin.y != 0.0 or origin.z != 0.0:
let world = f32_view(self.outputs.at(self.output_index("P")).bytes)
for at in 0..<points.length:
points[at] = Float3(x = (f32)((f64)world[at * 3] - origin.x), y = (f32)((f64)world[at * 3 + 1] - origin.y), z = (f32)((f64)world[at * 3 + 2] - origin.z))
let moved = try changed(source.point_span(), points)
defer free(moved) in memory.heap
return try moved_storage(source, self.positions, moved, keep_normals = true)
func output_index(name: str) -> usize:
for (at, output) in self.outputs.view().indexed():
if output.name == name: return at
return 0
## A written group as a group attribute.
func group_attribute(output: Output, source: const Mesh*) -> Attribute!:
_ = source
let words = u64[]((u64*)(try self.scratch.bytes((self.lanes + 63) // 64 * 8)).data, (self.lanes + 63) // 64)
for at in 0..<self.lanes:
if output.bytes[at] != 0: words[at >> 6] |= (u64)1 << (u64)(at & 63)
return try Attribute.group_of((str)output.name.bytes[6..], self.mode.domain(), self.lanes, words)
## A written detail value: Run Over Detail's one element, or for Numbers the highest
## lane whose value differs from what the detail held (the last writer).
func detail_attribute(output: Output) -> Attribute!:
var lane = self.lanes - 1
if self.lanes > 1:
let width = output.width
let size = output.kind.size()
var found = false
var at = self.lanes
while at > 1 and not found:
at -= 1
for byte in 0..<width * size:
if output.bytes[at * width * size + byte] != output.bytes[byte] and at > 0:
found = true
if found: lane = at
if not found: lane = 0
var numbers: f64[4]
for k in 0..<output.width:
numbers[k] = number_at(output.bytes, output.kind, lane * output.width + k)
let integer = output.kind == .i32 or output.kind == .i64
return try Attribute.from_numbers((str)output.name.bytes[7..], Domain.detail, output.width, numbers[..<output.width], integer, float32 = output.kind == .f32)
## The points whose positions changed, ascending: blocks of points counted on the pool,
## then filled in their places.
func changed(previous: const Float3[], next: const Float3[]) -> u32[]!:
let count = next.length if next.length < previous.length else previous.length
let blocks = (count + changed_block - 1) // changed_block
let starts = try new usize[blocks + 1] in memory.heap
defer free(starts) in memory.heap
var work = ChangedWork(previous = previous, next = next, count = count, starts = starts, moved = u32[]((u32*)starts.data, 0))
try parallel.for_each(blocks, (void*)&work, count_changed)
var total: usize = 0
for block in 0..<blocks:
let here = starts[block]
starts[block] = total
total += here
let moved = try new u32[total] --- in memory.heap
work.moved = moved
try parallel.for_each(blocks, (void*)&work, fill_changed)
return moved
let changed_block: usize = 65536
struct ChangedWork:
let previous: const Float3[]
let next: const Float3[]
let count: usize
let starts: usize[]
var moved: u32[]
func count_changed(context: void*, block: usize, worker: usize):
let work = (ChangedWork*)context
let first = block * changed_block
let end = first + changed_block if work.count - first > changed_block else work.count
var found: usize = 0
for at in first..<end:
if differs(work.previous[at], work.next[at]): found += 1
work.starts[block] = found
func fill_changed(context: void*, block: usize, worker: usize):
let work = (ChangedWork*)context
let first = block * changed_block
let end = first + changed_block if work.count - first > changed_block else work.count
var used = work.starts[block]
for at in first..<end:
if differs(work.previous[at], work.next[at]):
work.moved[used] = (u32)at
used += 1
func differs(a: Float3, b: Float3) -> bool:
return a.x.bits() != b.x.bits() or a.y.bits() != b.y.bits() or a.z.bits() != b.z.bits()
## Element `at` of `bytes` of `kind`, as f64.
func number_at(bytes: const u8[], kind: Kind, at: usize) -> f64:
return match kind:
.f32 => (f64)f32_view(bytes)[at]
.f64 => f64_view(bytes)[at]
.i32 => (f64)i32_view(bytes)[at]
.i64 => (f64)i64_view(bytes)[at]
_ => 1.0 if bytes[at] != 0 else 0.0
## The bits of `value` (or `default` bits when there is none) as a uniform of `kind`.
func number_bits(kind: Kind, value: f64?, default: u64) -> u64:
let number = value else return default
return match kind:
.f32 => (u64)((f32)number).bits()
.f64 => number.bits()
.i32 => (u64)(u32)(i32)number
.i64 => (u64)(i64)number
.u32 => (u64)(u32)number
.u64 => (u64)number
_ => 1 if number != 0.0 else 0
## `name[2]` as ("name", 2); a plain name as (name, 0).
func split_component(name: str) -> (str, usize):
if name.length >= 3 and name.bytes[name.length - 1] == ']':
var at = name.length - 1
while at > 0 and name.bytes[at] != '[': at -= 1
if at > 0:
var component: usize = 0
for byte in name.bytes[at + 1..<name.length - 1]:
component = component * 10 + (usize)(byte - '0')
return ((str)name.bytes[..<at], component)
return (name, 0)
## The attribute type that stores `kind`.
func data_type_of(kind: Kind) -> DataType?:
return match kind:
.f32 => DataType.float32
.f64 => DataType.float64
.i32 => DataType.int32
.i64 => DataType.int64
_ => none