Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-geocore

Source of release 0.7.5

0.7.5 / src / geometry_files / geometry.lucb

## The geometry codec: a whole GeometrySet as a prism crate (.prism), every
## component, attribute and group, exactly. Cooked geometry is written only
## by a Cache/Freeze node or an export; projects keep recipes.
##
##   /geometry            geometry   int64 version, int64 key
##     /geometry/a0       attribute  the set's detail attributes (as below)
##     /geometry/mesh     mesh       origin, P, face_offsets, corner_points,
##                                   triangles, [precise]
##       /geometry/mesh/a0  attribute  name, domain, type, width, single,
##                                     flags, role, elements, [table], value
##     /geometry/points   points     origin, P
##     /geometry/curves   curves     origin, P, offsets, [knots, knot_offsets]
##     /geometry/instances  instances  count, prototypes, prototype,
##                                     placement, visible
##       .../p0           geometry   (a prototype set, written once)
##     /geometry/volume   volume     count
##       .../g0           grid       name, class, background, transform,
##                                   keys, values, masks
##     /geometry/sdf      sdf        ops, params
##     /geometry/scene    scene      count, rows, paths (cameras, lights)
##     /geometry/materials  materials  count, paths, shaders, parameters, names, values, widths

## Each family is a codec (encode, decode) found by its component name;
## the built-in families are here, and other packages register theirs
## (luce-cad's B-rep). Columns are written as typed arrays straight from
## their blocks and read straight into new blocks. A column over 256 MiB is
## several properties (`<name>_chunks`: count and rows, then `<name>_0`…).
## Derived caches (normals, edges, evaluated curves, BVHs) are never written.

## What a family's codec does: write a component's properties and children
## under its element, and read them back into new component data.
pub struct ComponentCodec:
    pub let name: str
    pub let kind: const ComponentType*
    pub let encode: func(const void*, GeometryWriter*, str) -> unit!
    pub let decode: func(GeometryReader*, str) -> void*!

var codec_lock: sync.Mutex
var codecs: (ComponentCodec?)[64]
var codec_count: usize = 0

## Register the codec of a family (once, before its components are saved
## or loaded; registering again replaces it).
pub func register_codec(kind: const ComponentType*, encode: func(const void*, GeometryWriter*, str) -> unit!, decode: func(GeometryReader*, str) -> void*!) -> !:
    try register(kind)
    codec_lock.lock()
    defer codec_lock.unlock()
    for at in 0..<codec_count:
        let known = codecs[at] else continue
        if known.name == kind.name:
            codecs[at] = ComponentCodec(kind.name, kind, encode, decode)
            return
    if codec_count == codecs.length: error(errors.limit, "at most 64 geometry codecs")
    codecs[codec_count] = ComponentCodec(kind.name, kind, encode, decode)
    codec_count += 1

## The codec of the family called `name`: a built-in one, else a registered one.
func codec_of(name: str) -> ComponentCodec?:
    if name == "mesh": return ComponentCodec(name, mesh_component_type(), encode_mesh, decode_mesh)
    if name == "points": return ComponentCodec(name, points_component_type(), encode_points, decode_points)
    if name == "curves": return ComponentCodec(name, curves_type(), encode_curves, decode_curves)
    if name == "instances": return ComponentCodec(name, instances_component_type(), encode_instances, decode_instances)
    if name == "volume": return ComponentCodec(name, volume_type(), encode_volume, decode_volume)
    if name == "sdf": return ComponentCodec(name, sdf_type(), encode_sdf, decode_sdf)
    if name == "scene": return ComponentCodec(name, &scene_component, encode_scene, decode_scene)
    if name == "materials": return ComponentCodec(name, &materials_component, encode_materials, decode_materials)
    codec_lock.lock()
    defer codec_lock.unlock()
    for at in 0..<codec_count:
        let known = codecs[at] else continue
        if known.name == name: return known
    return none

func mesh_component_type() -> const ComponentType*:
    return &mesh_component

func points_component_type() -> const ComponentType*:
    return &points_component

func instances_component_type() -> const ComponentType*:
    return &instances_component

# mark: Writing

## Where codecs write: element and property calls on the crate being built,
## with the column helpers geometry needs.
##
## Payload lifetimes: the crate streams payloads when `save` runs, so every
## payload must outlive it. `column` copies its bytes (safe for temporaries,
## stack arrays and anything freed before the save); `column_of` retains a
## geocore Column until the writer closes (no copy); `column_borrowed`
## borrows bytes the caller keeps alive and unchanged until `save` returns
## (no copy: the built-in codecs use it for the columns of the set being
## saved, which the caller holds).
pub struct GeometryWriter:
    pub var crate: CrateWriter
    ## Path text built while nesting.
    var scratch: Growing[u8]
    ## Copied and retained payloads, alive until the writer closes.
    var kept: Growing[Column[u8]]
    ## The most bytes a property holds before a column is chunked.
    pub var chunk: usize = chunk_bytes

    pub func init():
        self.crate = CrateWriter()
        self.scratch = Growing[u8]()
        self.kept = Growing[Column[u8]]()
        self.chunk = chunk_bytes

    pub func close():
        self.crate.close()
        self.scratch.close()
        for at in 0..<self.kept.count: self.kept.items[at].release()
        self.kept.close()

    ## `parent` + "/" + `name`, valid until the next call.
    pub func child(parent: str, name: str) -> str!:
        try self.scratch.assign(parent.bytes)
        try self.scratch.push(47)
        for byte in name.bytes: try self.scratch.push(byte)
        return (str)self.scratch.view()

    ## A typed column of `rows` × `columns` components, its bytes COPIED now:
    ## safe whatever happens to them before the save. Chunked above 256 MiB.
    pub func column(name: str, dtype: u8, rows: usize, columns: usize, bytes: const u8[]) -> !:
        var copy = try Column[u8].copy_of(bytes)
        let view = copy.view()
        self.kept.push(copy) catch failure:
            copy.release()
            error(failure.code, failure.message)
        try self.column_borrowed(name, dtype, rows, columns, view)

    ## A typed column streamed from `column`, which the writer retains (its
    ## block's users + 1) until it closes: no copy.
    pub func column_of(name: str, dtype: u8, rows: usize, columns: usize, column: const Column[u8]*) -> !:
        var share = column.share()
        let view = share.view()
        self.kept.push(share) catch failure:
            share.release()
            error(failure.code, failure.message)
        try self.column_borrowed(name, dtype, rows, columns, view)

    ## A typed column BORROWING `bytes`, which must stay alive and unchanged
    ## until `save` returns: no copy. For columns of the set being saved.
    pub func column_borrowed(name: str, dtype: u8, rows: usize, columns: usize, bytes: const u8[]) -> !:
        if bytes.length <= self.chunk or rows <= 1:
            try self.crate.array_borrowed(name, dtype, rows, columns, bytes)
            return
        let row_bytes = bytes.length // rows
        let per_chunk = math.max(1.0, (f64)(self.chunk // row_bytes))
        let chunk_rows = (usize)per_chunk
        let chunks = (rows + chunk_rows - 1) // chunk_rows
        var header: i64[2] = [(i64)chunks, (i64)rows]
        var label = Growing[u8]()
        defer label.close()
        try append_name(&label, name, "_chunks")
        try self.crate.copied_array((str)label.view(), dtype_int64, 2, u8[]((u8*)&header[0], 16))
        for chunk in 0..<chunks:
            let first = chunk * chunk_rows
            let end = first + chunk_rows if first + chunk_rows < rows else rows
            label.clear()
            try append_name(&label, name, "_")
            try append_number(&label, chunk)
            try self.crate.array_borrowed((str)label.view(), dtype, end - first, columns, bytes[first * row_bytes..<end * row_bytes])

    ## Every attribute of `store` as children of `parent` (a0, a1, …).
    pub func attributes(parent: str, store: const AttributeStore*) -> !:
        var entries = Growing[const Attribute*]()
        defer entries.close()
        for index in 0..<store.count(): try entries.push(store.entry(index))
        try self.attribute_list(parent, entries.view())

    ## The attributes listed, as children of `parent`.
    pub func attribute_list(parent: str, entries: const (const Attribute*)[]) -> !:
        let base = try copied_text(parent)
        defer free(base) in memory.heap
        for index in 0..<entries.length:
            let entry = entries[index]
            var name = Growing[u8]()
            defer name.close()
            try append_text(&name, "a")
            try append_number(&name, index)
            let path = try self.child((str)base, (str)name.view())
            try self.crate.element(path, "attribute")
            try self.crate.text("name", entry.name())
            try self.crate.text("domain", domain_name(entry.domain))
            try self.crate.text("type", type_name(entry.kind))
            try self.crate.integer("width", (i64)entry.width)
            try self.crate.integer("single", 1 if entry.storage == Storage.single else 0)
            try self.crate.integer("flags", (i64)entry.flags)
            try self.crate.integer("role", (i64)(u8)entry.role)
            try self.crate.integer("elements", (i64)entry.elements)
            if entry.kind == DataType.text: try self.column_of("table", dtype_uint8, entry.table.count(), 1, &entry.table)
            if entry.storage == Storage.ragged:
                # An array attribute: its item ranges, then its items as the values.
                try self.crate.integer("ragged", 1)
                try self.column_of("offsets", dtype_int32, entry.offsets.count() // 4, 1, &entry.offsets)
            let data = entry.data.view()
            match entry.kind:
                .boolean: try self.column_of("value", dtype_uint64, data.length // 8, 1, &entry.data)
                .int8: try self.column_of("value", dtype_int8, data.length // entry.width, entry.width, &entry.data)
                .int32, .text: try self.column_of("value", dtype_int32, data.length // (4 * entry.width), entry.width, &entry.data)
                .int64: try self.column_of("value", dtype_int64, data.length // (8 * entry.width), entry.width, &entry.data)
                .float32: try self.column_of("value", dtype_float32, data.length // (4 * entry.width), entry.width, &entry.data)
                _: try self.column_of("value", dtype_float64, data.length // (8 * entry.width), entry.width, &entry.data)

func append_name(out: Growing[u8]*, name: str, suffix: str) -> !:
    try append_text(out, name)
    try append_text(out, suffix)

## The bytes of a typed slice, for a column payload (no copy).
func bytes_view[T](values: const T[]) -> const u8[]:
    return u8[]((u8*)values.data, values.length * memory.size_of(T))

## `set` under element `path` (kind "geometry"), its components as
## children; the root also records the format version and `key`.
pub func encode_set(writer: GeometryWriter*, set: const GeometrySet*, path: str, root: bool = false, key: i64 = 0) -> !:
    try writer.crate.element(path, "geometry")
    if root:
        try writer.crate.integer("version", 1)
        try writer.crate.integer("key", key)
    let base = try copied_text(path)
    defer free(base) in memory.heap
    try writer.attributes((str)base, set_detail(set))
    for index in 0..<set.component_count():
        let component = set.component(index)
        let codec = codec_of(component.kind.name) else error(errors.invalid, "this geometry family cannot be saved yet (no codec)")
        let child = try copied_text(try writer.child((str)base, component.kind.name))
        defer free(child) in memory.heap
        let encode = codec.encode
        try encode(component.data, writer, (str)child)

## A copy of `text` the caller frees (paths outlive the writer's scratch).
func copied_text(text: str) -> u8[]!:
    let copy = try new u8[text.length] --- in memory.heap
    if text.length > 0: memory.copy(copy, text.bytes, text.length)
    return copy

func encode_mesh(data: const void*, writer: GeometryWriter*, path: str) -> !:
    let mesh = (const Mesh*)data
    try writer.crate.element(path, "mesh")
    let origin = mesh.position_origin()
    try writer.crate.triple("origin", origin.x, origin.y, origin.z)
    let points = mesh.point_span()
    try writer.column_borrowed("P", dtype_float32, points.length, 3, bytes_view(points))
    let offsets = mesh.offset_span()
    try writer.column_borrowed("face_offsets", dtype_int32, offsets.length, 1, bytes_view(offsets))
    let corners = mesh.corner_span()
    try writer.column_borrowed("corner_points", dtype_int32, corners.length, 1, bytes_view(corners))
    let triangles = mesh.triangle_span()
    try writer.column_borrowed("triangles", dtype_int32, triangles.length, 1, bytes_view(triangles))
    let precise = precise_span(mesh)
    if precise.length > 0: try writer.column_borrowed("precise", dtype_float64, precise.length, 3, bytes_view(precise))
    var entries = Growing[const Attribute*]()
    defer entries.close()
    for index in 0..<(usize)mesh.attribute_count(): try entries.push(try mesh.attribute_entry((i64)index))
    try writer.attribute_list(path, entries.view())

func encode_points(data: const void*, writer: GeometryWriter*, path: str) -> !:
    let cloud = (const PointCloud*)data
    try writer.crate.element(path, "points")
    let origin = cloud_origin(cloud)
    try writer.crate.triple("origin", origin.x, origin.y, origin.z)
    let points = cloud_positions(cloud)
    try writer.column_borrowed("P", dtype_float32, points.length, 3, bytes_view(points))
    try writer.attributes(path, cloud_store(cloud))

func encode_curves(data: const void*, writer: GeometryWriter*, path: str) -> !:
    let curves = (const Curves*)data
    try writer.crate.element(path, "curves")
    let origin = curves_origin(curves)
    try writer.crate.triple("origin", origin.x, origin.y, origin.z)
    let points = curves.position_span()
    try writer.column_borrowed("P", dtype_float32, points.length, 3, bytes_view(points))
    let offsets = curves.offset_span()
    try writer.column_borrowed("offsets", dtype_int32, offsets.length, 1, bytes_view(offsets))
    let knots = curves_knots(curves)
    if knots.length > 0:
        try writer.column_borrowed("knots", dtype_float32, knots.length, 1, bytes_view(knots))
        let starts = curves_knot_offsets(curves)
        try writer.column_borrowed("knot_offsets", dtype_int32, starts.length, 1, bytes_view(starts))
    try writer.attributes(path, curves.store())

## Instances: the instances' prototypes, placements and visibility as
## columns, their row attributes (children a0, a1, …), then each prototype
## once (children p0, p1, …).
func encode_instances(data: const void*, writer: GeometryWriter*, path: str) -> !:
    let instances = (const Instances*)data
    try writer.crate.element(path, "instances")
    let count = instances.count()
    var prototypes = Growing[i32]()
    defer prototypes.close()
    var placements = Growing[f64]()
    defer placements.close()
    var visible = Growing[u8]()
    defer visible.close()
    for at in 0..<count:
        let row = instances.row(at)
        try prototypes.push(row.prototype)
        for value in [row.translation.x, row.translation.y, row.translation.z, row.rotation.x, row.rotation.y, row.rotation.z, row.scale.x, row.scale.y, row.scale.z]:
            try placements.push(value)
        try visible.push(1 if row.visible else 0)
    try writer.crate.integer("count", (i64)count)
    try writer.crate.integer("prototypes", (i64)instances.prototype_count())
    try writer.crate.copied_array("prototype", dtype_int32, count, u8[]((u8*)prototypes.items.data, count * 4))
    try writer.crate.copied_array("placement", dtype_float64, count * 9, u8[]((u8*)placements.items.data, count * 72))
    try writer.crate.copied_array("visible", dtype_uint8, count, visible.view())
    try writer.attributes(path, instances.store())
    let base = try copied_text(path)
    defer free(base) in memory.heap
    for index in 0..<instances.prototype_count():
        var name = Growing[u8]()
        defer name.close()
        try append_text(&name, "p")
        try append_number(&name, index)
        let nested = try copied_text(try writer.child((str)base, (str)name.view()))
        defer free(nested) in memory.heap
        try encode_set(writer, instances.prototype(index), (str)nested)

## Save `set` to `path` as a prism crate, `key` in its root (the Cache
## node keeps its stamp there); the bytes written.
pub func save_geometry(set: const GeometrySet*, path: str, key: i64, chunk: usize = chunk_bytes) -> usize!:
    var writer = GeometryWriter()
    defer writer.close()
    writer.chunk = chunk
    try encode_set(&writer, set, "/geometry", true, key)
    return try writer.crate.save(path)

func domain_name(domain: Domain) -> str:
    match domain:
        .point: return "point"
        .corner: return "corner"
        .face: return "face"
        .edge: return "edge"
        _: return "detail"

func type_name(kind: DataType) -> str:
    match kind:
        .boolean: return "boolean"
        .int8: return "int8"
        .int32: return "int32"
        .int64: return "int64"
        .float32: return "float32"
        .float64: return "float64"
        _: return "text"