dymokomi/luce-geocore
Source of release 0.7.8
0.7.8 / src / geometry_files / crate.lucb
## Prism v4 binary crates (PRSMC), written and read as streams. The format is
## luce-prism's (docs/FORMAT.md, serialization/binary.lucb): the magic, u16
## version 4, u16 flags, a string table, elements with typed properties, then
## the (here empty) connection, deletion and disconnection records. A value is
## u8 dtype, u8 rank, u32 dimensions, then its row-major payload.
##
## luce-prism builds a whole document in memory and caps a file at 256 MiB.
## Geometry columns are large, so this writer streams each payload straight
## from its column, and the reader reads a payload straight into the column
## that will hold it. Files under 256 MiB read back in luce-prism unchanged.
## Floats are written bit for bit (luce-prism zeroes non-finite ones), so a
## geometry round trip is exact. Compressed crates are not read here.
import c
from luce_std import files
import interop
from luce_std import math
import memory
import strings
import sync
import errors
from core.growing import Growing
from core.shared import Column
from core.float3 import Float3
from core.attributes import Attribute, AttributeStore, Domain, DataType, Storage, Role
from math.vector import Vector3
from math.matrix import Matrix4
from geometries.polygon import Mesh, assembled, put_attribute, mesh_dispose, precise_span
from geometry_set import GeometrySet, ComponentType, Component, Placement, PointCloud, Instances, mesh_component, points_component, instances_component, empty_storage, dispose_set, adopt_set, append_text, append_number, new_instances, cloud_origin, cloud_positions, cloud_store, cloud_assembled, register, set_detail, set_put_detail, SceneObjects, SceneKind, Projection, LightShape, Camera, Light, scene_component, new_scene_objects, camera_object, light_object, Materials, materials_component, new_materials
from curves import Curves, CurvesExact, curves_type, curves_origin, curves_assembled, curves_knots, curves_knot_offsets
from fields import Volume, Sdf, GridClass, VolumeLook, new_grid, new_volume, add_grid, release_volume, new_sdf, volume_type, sdf_type, leaf_voxels, mask_words, sdf_stride
## Wire dtype codes (luce-prism's DType; never reordered): booleans packed as
## bits, then signed and unsigned integers, floats and strings.
pub let dtype_bool: u8 = 0
## 8-bit signed integers.
pub let dtype_int8: u8 = 2
## 32-bit signed integers.
pub let dtype_int32: u8 = 4
## 64-bit signed integers.
pub let dtype_int64: u8 = 5
## 8-bit unsigned integers (raw bytes).
pub let dtype_uint8: u8 = 6
## 64-bit unsigned integers (bit words).
pub let dtype_uint64: u8 = 9
## 32-bit floats.
pub let dtype_float32: u8 = 10
## 64-bit floats.
pub let dtype_float64: u8 = 11
## Strings from the crate's string table.
pub let dtype_str: u8 = 12
## Bytes per component of a fixed-width dtype (0 for str).
pub func dtype_width(dtype: u8) -> usize:
match dtype:
0, 1, 2, 6: return 1
3, 7, 13: return 2
4, 8, 10: return 4
5, 9, 11: return 8
_: return 0
## The most bytes one property's payload holds; bigger columns are written as
## several properties (see geometry.lucb), so a streaming prism reader with
## luce-prism's 256 MiB buffer limit can take each.
pub let chunk_bytes: usize = 268435456
## A span of a text pool.
struct Text:
let first: u32
let length: u32
struct PropertyPlan:
let name: u32
let dtype: u8
let rank: u8
let dims: u32[3]
## Borrowed column bytes, or bytes of the writer's inline pool.
let payload: const u8[]
let inline_first: usize
let inline_length: usize
let inline: bool
struct ElementPlan:
let path: u32
let kind: u32
let first: usize
var count: usize
## Collects a document of elements and properties, then streams it to a
## file. Payloads are borrowed until `save` returns.
pub struct CrateWriter:
var pool: Growing[u8]
var texts: Growing[Text]
## Open addressing over `texts`: 0 empty, else index + 1.
var slots: Growing[u32]
var inline: Growing[u8]
var elements: Growing[ElementPlan]
var properties: Growing[PropertyPlan]
## An empty document.
pub func init():
self.pool = Growing[u8]()
self.texts = Growing[Text]()
self.slots = Growing[u32]()
self.inline = Growing[u8]()
self.elements = Growing[ElementPlan]()
self.properties = Growing[PropertyPlan]()
## Release the document's buffers.
pub func close():
self.pool.close()
self.texts.close()
self.slots.close()
self.inline.close()
self.elements.close()
self.properties.close()
func text_at(index: u32) -> str:
let text = self.texts.items[(usize)index]
return (str)self.pool.items[(usize)text.first..<(usize)(text.first + text.length)]
## The string table index of `text`, added when new.
func intern(text: str) -> u32!:
if (self.texts.count + 1) * 2 > self.slots.count:
var size: usize = 64
while size < (self.texts.count + 1) * 4: size *= 2
try self.slots.fill(size, 0)
for index in 0..<self.texts.count: self.slots.items[self.slot_for(self.text_at((u32)index))] = (u32)index + 1
let slot = self.slot_for(text)
if self.slots.items[slot] != 0: return self.slots.items[slot] - 1
let first = self.pool.count
try self.pool.reserve(first + text.length)
if text.length > 0: memory.copy(self.pool.items[first..], text.bytes, text.length)
self.pool.count += text.length
try self.texts.push(Text((u32)first, (u32)text.length))
let index = (u32)(self.texts.count - 1)
self.slots.items[slot] = index + 1
return index
func slot_for(text: str) -> usize:
var value: u64 = 14695981039346656037
for byte in text.bytes: value = (value ^ (u64)byte) *% 1099511628211
let mask = self.slots.count - 1
var at = (usize)value & mask
while self.slots.items[at] != 0 and self.text_at(self.slots.items[at] - 1) != text:
at = (at + 1) & mask
return at
## Start element `path` of `kind`; properties follow until the next.
pub func element(path: str, kind: str) -> !:
let path_index = try self.intern(path)
let kind_index = try self.intern(kind)
try self.elements.push(ElementPlan(path_index, kind_index, self.properties.count, 0))
## An array property BORROWING `payload` (rank 1 or 2: `rows`, and
## `columns` when above 1): the bytes must stay alive and unchanged until
## `save` returns. Callers with temporaries use GeometryWriter.column,
## which copies, or copied_array.
pub func array_borrowed(name: str, dtype: u8, rows: usize, columns: usize, payload: const u8[]) -> !:
if rows > 4294967295: error(errors.limit, "a prism dimension exceeds 2^32")
let rank: u8 = 2 if columns > 1 else 1
try self.add(PropertyPlan(try self.intern(name), dtype, rank, [(u32)rows, (u32)columns, 0], payload, 0, 0, false))
## A scalar property whose bytes are copied now.
pub func scalar(name: str, dtype: u8, bytes: const u8[]) -> !:
let first = self.inline.count
try self.inline.reserve(first + bytes.length)
memory.copy(self.inline.items[first..], bytes, bytes.length)
self.inline.count += bytes.length
var empty: const u8[]
try self.add(PropertyPlan(try self.intern(name), dtype, 0, [0, 0, 0], empty, first, bytes.length, true))
## A scalar i64 property of the current element.
pub func integer(name: str, value: i64) -> !:
var copy = value
try self.scalar(name, dtype_int64, u8[]((u8*)©, 8))
## A scalar f64 property of the current element.
pub func number(name: str, value: f64) -> !:
var copy = value
try self.scalar(name, dtype_float64, u8[]((u8*)©, 8))
## Three f64 components (a translation, an origin).
pub func triple(name: str, x: f64, y: f64, z: f64) -> !:
let values: f64[3] = [x, y, z]
try self.copied_array(name, dtype_float64, 3, u8[]((u8*)&values[0], 24))
## A small rank-1 array whose bytes are copied now.
pub func copied_array(name: str, dtype: u8, count: usize, bytes: const u8[]) -> !:
let first = self.inline.count
try self.inline.reserve(first + bytes.length)
memory.copy(self.inline.items[first..], bytes, bytes.length)
self.inline.count += bytes.length
var empty: const u8[]
try self.add(PropertyPlan(try self.intern(name), dtype, 1, [(u32)count, 0, 0], empty, first, bytes.length, true))
## A str scalar: u32 length and the bytes.
pub func text(name: str, value: str) -> !:
var encoded: u8[4] = [(u8)(value.length & 255), (u8)((value.length >> 8) & 255), (u8)((value.length >> 16) & 255), (u8)((value.length >> 24) & 255)]
let first = self.inline.count
try self.inline.reserve(first + 4 + value.length)
memory.copy(self.inline.items[first..], encoded, 4)
if value.length > 0: memory.copy(self.inline.items[first + 4..], value.bytes, value.length)
self.inline.count += 4 + value.length
var empty: const u8[]
try self.add(PropertyPlan(try self.intern(name), dtype_str, 0, [0, 0, 0], empty, first, 4 + value.length, true))
func add(property: PropertyPlan) -> !:
if self.elements.count == 0: error(errors.invalid, "a prism property needs an element")
try self.properties.push(property)
self.elements.items[self.elements.count - 1].count += 1
## Write the document to `path`: through a temporary file in the same
## directory, renamed over `path` when complete. Returns the bytes.
pub func save(path: str) -> usize!:
let directory = try strings.copy(parent_directory(path))
defer strings.release(directory)
let destination = try strings.copy(path)
defer strings.release(destination)
var file = try files.TemporaryFile.create((c.str)directory, 420)
errdefer file.destroy()
var out = try Sink.make(&file)
defer out.close()
try out.text("PRSMC\0")
try out.uint(4, 2)
try out.uint(0, 2)
try out.uint((u64)self.texts.count, 4)
for index in 0..<self.texts.count:
let text = self.text_at((u32)index)
try out.uint((u64)text.length, 4)
try out.text(text)
try out.uint((u64)self.elements.count, 4)
for element in self.elements.view():
try out.uint((u64)element.path, 4)
try out.uint((u64)element.kind, 4)
try out.uint((u64)element.count, 4)
for property in self.properties.view()[element.first..<element.first + element.count]:
try out.uint((u64)property.name, 4)
try out.uint((u64)property.dtype, 1)
try out.uint((u64)property.rank, 1)
for dimension in 0..<(usize)property.rank: try out.uint((u64)property.dims[dimension], 4)
if property.inline: try out.put(self.inline.items[property.inline_first..<property.inline_first + property.inline_length])
else: try out.put(property.payload)
# Not animated; no catalog type name.
try out.uint(0, 1)
try out.uint(0, 4)
# No metadata, not an override, no property or metadata tombstones.
try out.uint(0, 4)
try out.uint(0, 1)
try out.uint(0, 4)
try out.uint(0, 4)
# No connections, deletions or disconnections; no tagged sections.
try out.uint(0, 4)
try out.uint(0, 4)
try out.uint(0, 4)
try out.flush()
let written = out.written
try file.publish((c.str)destination, true)
return written
## The directory part of `path` ("." when it has none).
func parent_directory(path: str) -> str:
var at = path.length
while at > 0:
at -= 1
if path.bytes[at] == 47: return "/" if at == 0 else (str)path.bytes[..<at]
return "."
## Buffered output: small writes gather in a 1 MiB buffer; large payloads go
## straight to the file.
struct Sink:
let file: files.TemporaryFile*
var buffer: u8[]
var used: usize = 0
var written: usize = 0
static func make(file: files.TemporaryFile*) -> Sink!:
return Sink(file, try new u8[1048576] --- in memory.heap, 0, 0)
func put(data: const u8[]) -> !:
if data.length > self.buffer.length - self.used:
try self.flush()
if data.length >= self.buffer.length:
try write_all(self.file, data)
self.written += data.length
return
memory.copy(self.buffer[self.used..], data, data.length)
self.used += data.length
self.written += data.length
func text(value: str) -> !:
try self.put(value.bytes)
func uint(value: u64, width: usize) -> !:
var bytes: u8[8] = ---
for at in 0..<width: bytes[at] = (u8)((value >> (u64)(at * 8)) & 255)
try self.put(bytes[..<width])
func flush() -> !:
if self.used > 0: try write_all(self.file, self.buffer[..<self.used])
self.used = 0
func close():
free(self.buffer) in memory.heap
func write_all(file: files.TemporaryFile*, data: const u8[]) -> !:
var at: usize = 0
while at < data.length:
let count = try file.write(data[at..])
if count == 0: error(errors.invalid, "the geometry file could not be written")
at += count
# mark: Reading
## A property's header: its name, dtype and shape, and its payload's bytes.
pub struct PropertyHeader:
## The property's name.
pub let name: str
## Its wire dtype.
pub let dtype: u8
## How many dimensions its shape has (0: a scalar).
pub let rank: u8
## The shape's dimensions.
pub let dims: u64[4]
## Its payload's bytes.
pub let bytes: usize
## Components: the product of the dimensions (1 for a scalar).
pub func components() -> u64:
var total: u64 = 1
for at in 0..<(usize)self.rank: total *= self.dims[at]
return total
## Reads a crate element by element and property by property. Paths, kinds
## and names are views of the string table, valid while the reader is.
pub struct CrateReader:
var file: files.File
var buffer: u8[]
var start: usize = 0
var end: usize = 0
var pool: Growing[u8]
var texts: Growing[Text]
## Elements left, and properties left in the current element.
var elements_left: usize = 0
var properties_left: usize = 0
## A property whose payload is still unread.
var pending: usize = 0
var in_element: bool = false
## Open the crate at `path` and read its string table.
pub static func open(path: str) -> CrateReader!:
let native = try strings.copy(path)
defer strings.release(native)
var file = try files.File.open((c.str)native)
errdefer file.destroy()
let buffer = try new u8[1048576] --- in memory.heap
var reader = CrateReader(file, buffer, 0, 0, Growing[u8](), Growing[Text]())
errdefer reader.close()
let magic = try reader.take_bytes(6)
if (str)magic != "PRSMC\0": error(errors.invalid, "not a prism binary crate")
if try reader.uint(2) != 4: error(errors.invalid, "only prism v4 crates are supported")
if try reader.uint(2) != 0: error(errors.invalid, "compressed prism crates are not supported for geometry")
let count = (usize)(try reader.uint(4))
for index in 0..<count:
let length = (usize)(try reader.uint(4))
if length > 1048576: error(errors.invalid, "implausible prism string")
let first = reader.pool.count
try reader.pool.reserve(first + length)
try reader.read_exact(reader.pool.items[first..<first + length])
reader.pool.count += length
try reader.texts.push(Text((u32)first, (u32)length))
reader.elements_left = (usize)(try reader.uint(4))
return reader
## Close the file and release the buffers.
pub func close():
self.file.destroy()
free(self.buffer) in memory.heap
self.pool.close()
self.texts.close()
func text_at(index: u64) -> str!:
if index >= (u64)self.texts.count: error(errors.invalid, "invalid prism string index")
let text = self.texts.items[(usize)index]
return (str)self.pool.items[(usize)text.first..<(usize)(text.first + text.length)]
## The next element's path and kind, or none after the last.
pub func next_element() -> (str, str)?!:
try self.finish_element()
if self.elements_left == 0: return none
self.elements_left -= 1
let path = try self.text_at(try self.uint(4))
let kind = try self.text_at(try self.uint(4))
self.properties_left = (usize)(try self.uint(4))
self.in_element = true
return (path, kind)
## The current element's next property, or none after its last.
pub func next_property() -> PropertyHeader?!:
try self.skip_payload()
if self.properties_left == 0: return none
self.properties_left -= 1
let name = try self.text_at(try self.uint(4))
let dtype = (u8)(try self.uint(1))
let rank = (u8)(try self.uint(1))
if rank > 4: error(errors.invalid, "prism values have at most 4 dimensions here")
var dims: u64[4] = [1, 1, 1, 1]
var components: u64 = 1
for at in 0..<(usize)rank:
dims[at] = try self.uint(4)
components *= dims[at]
var bytes: usize = 0
if dtype == dtype_str:
# Strings are only read as scalars: their length follows.
if rank != 0: error(errors.invalid, "string arrays are not geometry")
bytes = (usize)(try self.peek_uint(4)) + 4
else:
let width = dtype_width(dtype)
if width == 0: error(errors.invalid, "unknown prism dtype")
bytes = (usize)components * width
self.pending = bytes + 1
return PropertyHeader(name, dtype, rank, dims, bytes)
## The current property's payload into `into` (exactly its bytes).
pub func read_payload(into: u8[]) -> !:
if self.pending == 0 or into.length != self.pending - 1: error(errors.invalid, "prism payload size mismatch")
try self.read_exact(into)
self.pending = 0
try self.finish_property()
## A scalar or small payload into a fixed buffer.
pub func read_small(into: u8[]) -> !:
try self.read_payload(into)
## A str scalar (copied into `out`).
pub func read_text(out: Growing[u8]*) -> !:
let length = (usize)(try self.uint(4))
try out.reserve(length)
try self.read_exact(out.items[..<length])
out.count = length
self.pending = 0
try self.finish_property()
func skip_payload() -> !:
if self.pending == 0: return
var left = self.pending - 1
while left > 0:
let available = self.end - self.start
if available == 0: try self.fill()
let step = left if left < self.end - self.start else self.end - self.start
self.start += step
left -= step
self.pending = 0
try self.finish_property()
## The animation flag (none here) and the catalog type name.
func finish_property() -> !:
if try self.uint(1) != 0: error(errors.invalid, "animated geometry properties are not supported")
let length = (usize)(try self.uint(4))
_ = try self.take_bytes(length)
## The rest of the current element: metadata, override flag, tombstones.
func finish_element() -> !:
if not self.in_element: return
while self.properties_left > 0 or self.pending > 0:
_ = try self.next_property()
try self.skip_payload()
let metadata = (usize)(try self.uint(4))
for at in 0..<metadata * 2: _ = try self.uint(4)
_ = try self.uint(1)
for list in 0..<(usize)2:
let count = (usize)(try self.uint(4))
for at in 0..<count: _ = try self.uint(4)
self.in_element = false
func fill() -> !:
if self.start > 0:
let kept = self.end - self.start
if kept > 0: memory.move(self.buffer[..<kept], self.buffer[self.start..<self.end], kept)
self.start = 0
self.end = kept
let count = try self.file.read(self.buffer[self.end..])
if count == 0: error(errors.invalid, "truncated prism crate")
self.end += count
## Bytes from the buffer, reading more as needed (at most its size).
func take_bytes(count: usize) -> const u8[]!:
if count > self.buffer.length: error(errors.invalid, "implausible prism record")
while self.end - self.start < count: try self.fill()
let first = self.start
self.start += count
return self.buffer[first..<first + count]
func uint(width: usize) -> u64!:
let bytes = try self.take_bytes(width)
var value: u64 = 0
for at in 0..<width: value = value | ((u64)bytes[at] << (u64)(at * 8))
return value
func peek_uint(width: usize) -> u64!:
while self.end - self.start < width: try self.fill()
var value: u64 = 0
for at in 0..<width: value = value | ((u64)self.buffer[self.start + at] << (u64)(at * 8))
return value
## Exactly `into.length` bytes: what is buffered, then straight from the
## file into `into`.
func read_exact(into: u8[]) -> !:
var at: usize = 0
let buffered = self.end - self.start
if buffered > 0:
let step = buffered if buffered < into.length else into.length
memory.copy(into[..<step], self.buffer[self.start..<self.start + step], step)
self.start += step
at = step
while at < into.length:
let count = try self.file.read(into[at..])
if count == 0: error(errors.invalid, "truncated prism crate")
at += count