dymokomi/luce-usd
Source of release 0.9.2
0.9.2 / src / usdc / reader / reader.lucb
## The usdc reader: crate files into LayerData, after OpenUSD's
## pxr/usd/sdf/crateFile.cpp. A crate is a bootstrap header (PXR-USDC, the
## version, the table of contents' offset), the values, then six structural
## sections: TOKENS, STRINGS, FIELDS (a token and a ValueRep each),
## FIELDSETS (runs of field indexes), PATHS (a tree) and SPECS (path, field
## set, spec type). A ValueRep is 64 bits: array, inlined, compressed and
## array-edit flags, a type byte and a 48-bit payload (an inline value or a
## file offset).
##
## Every read is checked against the file's bytes, so a damaged crate fails
## with `invalid` rather than reading out of bounds. Array values decode in
## parallel after the rest, each straight into its column.
##
## Fragments, in ORDER:
##
## reader.lucb the Crate state, the entry point, bounded reads
## sections.lucb bootstrap, table of contents, tokens, strings, fields,
## field sets and specs
## paths.lucb the path tree (compressed from 0.4.0, headers before)
## values.lucb ValueReps: inline values, dictionaries, list ops, time
## samples and the other structured values
## arrays.lucb arrays, compressed integers and floats, parallel decode
## splines.lucb spline values
## array_edits.lucb array edit values
import memory
import errors
import usdc.compression
from sdf.names import Names, PathKind, absolute_root, empty_path
from sdf.layer_data import LayerData, SpecType
from sdf.values import Value, Dictionary, ListOp, ListItem, ListPart, TimeSamples, Spline, Knot, CurveType, Interpolation, ExtrapolationMode, Extrapolation, TangentAlgorithm, ArrayEdit, EditInstruction, variability_uniform
from sdf.splines import settle
from sdf.value_types import ValueKind, element_bytes, components, scalar_of
from luce_geocore.core.shared import Column
from luce_geocore.core.growing import Growing
from luce_std import parallel
## The newest crate version this reader reads (major 0, minor 15).
pub let newest_minor: u8 = 15
## The message of the last failed read on this thread.
local var message_text: u8[320]
## Read the crate `data` into `layer` (an empty LayerData over the Names
## every layer of the load shares).
pub func read_usdc(data: const u8[], layer: LayerData*) -> !:
var crate = Crate(data = data, names = layer.names, minor = 0, patch = 0, tokens = Growing[u32](), strings = Growing[u32](), field_tokens = Growing[u32](), field_reps = Growing[u64](), field_sets = Growing[u32](), paths = Growing[u32](), specs = Growing[CrateSpec](), arrays = Growing[PendingArray](), depth = 0)
defer crate.close()
try crate.read_structure()
try crate.fill(layer)
## One spec: crate path index, field set index, spec type.
struct CrateSpec:
var path: u32
var field_set: u32
var kind: u32
## An array value decoded after the rest: where it goes and its rep.
struct PendingArray:
var spec: usize
var field: u32
var rep: u64
var value: Value
var failed: bool
var failure: ErrorCode
struct Crate:
var data: const u8[]
var names: Names*
var minor: u8
var patch: u8
## Crate token index to token id, string index to token id.
var tokens: Growing[u32]
var strings: Growing[u32]
## Fields: each one's name (a token id) and ValueRep.
var field_tokens: Growing[u32]
var field_reps: Growing[u64]
## Field indexes, each set ended by ~0.
var field_sets: Growing[u32]
## Crate path index to path id.
var paths: Growing[u32]
var specs: Growing[CrateSpec]
var arrays: Growing[PendingArray]
## Nesting of values inside values (dictionaries, VtValues), bounded.
var depth: usize
func close():
for index in 0..<self.arrays.count: self.arrays.items[index].value.release()
self.tokens.close()
self.strings.close()
self.field_tokens.close()
self.field_reps.close()
self.field_sets.close()
self.paths.close()
self.specs.close()
self.arrays.close()
## Whether the file's version is at least 0.minor.patch.
func since(minor: u8, patch: u8 = 0) -> bool:
return self.minor > minor or (self.minor == minor and self.patch >= patch)
## `count` bytes at `at`, checked.
func bytes_at(at: u64, count: u64) -> const u8[]!:
if at > (u64)self.data.length or count > (u64)self.data.length - at:
error(errors.invalid, "a crate file read past its end")
return self.data[(usize)at..<(usize)(at + count)]
func u64_at(at: u64) -> u64!:
return memory.read[u64]((void*)&(try self.bytes_at(at, 8))[0])
func i64_at(at: u64) -> i64!:
return memory.read[i64]((void*)&(try self.bytes_at(at, 8))[0])
func u32_at(at: u64) -> u32!:
return memory.read[u32]((void*)&(try self.bytes_at(at, 4))[0])
func f64_at(at: u64) -> f64!:
return memory.read[f64]((void*)&(try self.bytes_at(at, 8))[0])
## The token id of crate token `index`.
func token(index: u64) -> u32!:
if index >= (u64)self.tokens.count: error(errors.invalid, "a token index outside the crate's tokens")
return self.tokens.items[(usize)index]
## The token id of crate string `index`.
func string(index: u64) -> u32!:
if index >= (u64)self.strings.count: error(errors.invalid, "a string index outside the crate's strings")
return self.strings.items[(usize)index]
## The path id of crate path `index`; entries the path tree never
## fills stand for the empty path (a reference without a prim path).
func path(index: u64) -> u32!:
if index >= (u64)self.paths.count: error(errors.invalid, "a path index outside the crate's paths")
return self.paths.items[(usize)index]
## Specs and their fields into `layer`; arrays decode last, in parallel.
func fill(layer: LayerData*) -> !:
for crate_spec in self.specs.view():
let path = try self.path((u64)crate_spec.path)
if path == empty_path: error(errors.invalid, "a crate spec with an empty path")
if crate_spec.kind == 0 or crate_spec.kind > 11: error(errors.invalid, "a crate spec of an unknown type")
var spec: usize = 0
if path == absolute_root:
spec = 0
else:
if layer.find(path) != none: error(errors.invalid, "a crate spec repeated")
spec = try layer.create_spec(path, (SpecType)(u8)crate_spec.kind)
var at = (usize)crate_spec.field_set
while true:
if at >= self.field_sets.count: error(errors.invalid, "a field set runs past the crate's field sets")
let field = self.field_sets.items[at]
if field == 0xFFFFFFFF: break
if (usize)field >= self.field_reps.count: error(errors.invalid, "a field index outside the crate's fields")
let name = self.field_tokens.items[(usize)field]
let rep = self.field_reps.items[(usize)field]
if rep_array(rep) and rep_payload(rep) != 0:
# Decoded below; the field keeps its place in the spec.
try layer.set(spec, name, Value.marker(ValueKind.packed))
try self.arrays.push(PendingArray(spec = spec, field = name, rep = rep, value = Value.marker(ValueKind.empty), failed = false, failure = errors.invalid))
else:
try layer.set(spec, name, try self.value(rep))
at += 1
if self.arrays.count > 0:
try parallel.for_each(self.arrays.count, (void*)&self, decode_pending)
for index in 0..<self.arrays.count:
let pending = &self.arrays.items[index]
if pending.failed:
if pending.failure == memory.exhausted: error(memory.exhausted, "out of memory decoding a crate array")
error(pending.failure, "a damaged array value in a crate file")
try layer.set(pending.spec, pending.field, pending.value)
pending.value = Value.marker(ValueKind.empty)
## Decode one pending array on the pool.
func decode_pending(context: void*, index: usize, worker: usize):
let raw = context
let crate = (Crate*)raw
let pending = &crate.arrays.items[index]
pending.value = crate.array(pending.rep) catch failure:
pending.failed = true
pending.failure = failure.code
recover Value.marker(ValueKind.empty)
## ValueRep fields.
inline func rep_array(rep: u64) -> bool:
return (rep >> 63) & 1 != 0
inline func rep_inlined(rep: u64) -> bool:
return (rep >> 62) & 1 != 0
inline func rep_compressed(rep: u64) -> bool:
return (rep >> 61) & 1 != 0
inline func rep_array_edit(rep: u64) -> bool:
return (rep >> 60) & 1 != 0
inline func rep_type(rep: u64) -> u8:
return (u8)((rep >> 48) & 0xFF)
inline func rep_payload(rep: u64) -> u64:
return rep & 0xFFFFFFFFFFFF