Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-usd

Source of release 0.9.3

0.9.3 / src / usdc / reader / reader.lucb

## The usdc reader: crate files into LayerData, after OpenUSD's
## pxr/usd/sdf/crateFile.cpp. A crate is a bootstrap header (PXR-USDC, the
## version, the table of contents' offset), the values, then six structural
## sections: TOKENS, STRINGS, FIELDS (a token and a ValueRep each),
## FIELDSETS (runs of field indexes), PATHS (a tree) and SPECS (path, field
## set, spec type). A ValueRep is 64 bits: array, inlined, compressed and
## array-edit flags, a type byte and a 48-bit payload (an inline value or a
## file offset).
##
## Every read is checked against the file's bytes, so a damaged crate fails
## with `invalid` rather than reading out of bounds. Array values decode in
## parallel after the rest, each straight into its column.
##
## Fragments, in ORDER:
##
## reader.lucb    the Crate state, the entry point, bounded reads
## sections.lucb  bootstrap, table of contents, tokens, strings, fields,
##                field sets and specs
## paths.lucb     the path tree (compressed from 0.4.0, headers before)
## values.lucb    ValueReps: inline values, dictionaries, list ops, time
##                samples and the other structured values
## arrays.lucb    arrays, compressed integers and floats, parallel decode
## splines.lucb   spline values
## array_edits.lucb  array edit values
import memory
import errors
import usdc.compression
from sdf.names import Names, PathKind, absolute_root, empty_path
from sdf.layer_data import LayerData, SpecType
from sdf.values import Value, Dictionary, ListOp, ListItem, ListPart, TimeSamples, Spline, Knot, CurveType, Interpolation, ExtrapolationMode, Extrapolation, TangentAlgorithm, ArrayEdit, EditInstruction, variability_uniform
from sdf.splines import settle
from sdf.value_types import ValueKind, element_bytes, components, scalar_of
from luce_geocore.core.shared import Column
from luce_geocore.core.growing import Growing
from luce_std import parallel

## The newest crate version this reader reads (major 0, minor 15).
pub let newest_minor: u8 = 15

## The message of the last failed read on this thread.
local var message_text: u8[320]

## Read the crate `data` into `layer` (an empty LayerData over the Names
## every layer of the load shares).
pub func read_usdc(data: const u8[], layer: LayerData*) -> !:
    var crate = Crate(data = data, names = layer.names, minor = 0, patch = 0, tokens = Growing[u32](), strings = Growing[u32](), field_tokens = Growing[u32](), field_reps = Growing[u64](), field_sets = Growing[u32](), paths = Growing[u32](), specs = Growing[CrateSpec](), arrays = Growing[PendingArray](), depth = 0)
    defer crate.close()
    try crate.read_structure()
    try crate.fill(layer)

## One spec: crate path index, field set index, spec type.
struct CrateSpec:
    var path: u32
    var field_set: u32
    var kind: u32

## An array value decoded after the rest: where it goes and its rep.
struct PendingArray:
    var spec: usize
    var field: u32
    var rep: u64
    var value: Value
    var failed: bool
    var failure: ErrorCode

struct Crate:
    var data: const u8[]
    var names: Names*
    var minor: u8
    var patch: u8
    ## Crate token index to token id, string index to token id.
    var tokens: Growing[u32]
    var strings: Growing[u32]
    ## Fields: each one's name (a token id) and ValueRep.
    var field_tokens: Growing[u32]
    var field_reps: Growing[u64]
    ## Field indexes, each set ended by ~0.
    var field_sets: Growing[u32]
    ## Crate path index to path id.
    var paths: Growing[u32]
    var specs: Growing[CrateSpec]
    var arrays: Growing[PendingArray]
    ## Nesting of values inside values (dictionaries, VtValues), bounded.
    var depth: usize

    func close():
        for index in 0..<self.arrays.count: self.arrays.items[index].value.release()
        self.tokens.close()
        self.strings.close()
        self.field_tokens.close()
        self.field_reps.close()
        self.field_sets.close()
        self.paths.close()
        self.specs.close()
        self.arrays.close()

    ## Whether the file's version is at least 0.minor.patch.
    func since(minor: u8, patch: u8 = 0) -> bool:
        return self.minor > minor or (self.minor == minor and self.patch >= patch)

    ## `count` bytes at `at`, checked.
    func bytes_at(at: u64, count: u64) -> const u8[]!:
        if at > (u64)self.data.length or count > (u64)self.data.length - at:
            error(errors.invalid, "a crate file read past its end")
        return self.data[(usize)at..<(usize)(at + count)]

    func u64_at(at: u64) -> u64!:
        return memory.read[u64]((void*)&(try self.bytes_at(at, 8))[0])

    func i64_at(at: u64) -> i64!:
        return memory.read[i64]((void*)&(try self.bytes_at(at, 8))[0])

    func u32_at(at: u64) -> u32!:
        return memory.read[u32]((void*)&(try self.bytes_at(at, 4))[0])

    func f64_at(at: u64) -> f64!:
        return memory.read[f64]((void*)&(try self.bytes_at(at, 8))[0])

    ## The token id of crate token `index`.
    func token(index: u64) -> u32!:
        if index >= (u64)self.tokens.count: error(errors.invalid, "a token index outside the crate's tokens")
        return self.tokens.items[(usize)index]

    ## The token id of crate string `index`.
    func string(index: u64) -> u32!:
        if index >= (u64)self.strings.count: error(errors.invalid, "a string index outside the crate's strings")
        return self.strings.items[(usize)index]

    ## The path id of crate path `index`; entries the path tree never
    ## fills stand for the empty path (a reference without a prim path).
    func path(index: u64) -> u32!:
        if index >= (u64)self.paths.count: error(errors.invalid, "a path index outside the crate's paths")
        return self.paths.items[(usize)index]

    ## Specs and their fields into `layer`; arrays decode last, in parallel.
    func fill(layer: LayerData*) -> !:
        for crate_spec in self.specs.view():
            let path = try self.path((u64)crate_spec.path)
            if path == empty_path: error(errors.invalid, "a crate spec with an empty path")
            if crate_spec.kind == 0 or crate_spec.kind > 11: error(errors.invalid, "a crate spec of an unknown type")
            var spec: usize = 0
            if path == absolute_root:
                spec = 0
            else:
                if layer.find(path) != none: error(errors.invalid, "a crate spec repeated")
                spec = try layer.create_spec(path, (SpecType)(u8)crate_spec.kind)
            var at = (usize)crate_spec.field_set
            while true:
                if at >= self.field_sets.count: error(errors.invalid, "a field set runs past the crate's field sets")
                let field = self.field_sets.items[at]
                if field == 0xFFFFFFFF: break
                if (usize)field >= self.field_reps.count: error(errors.invalid, "a field index outside the crate's fields")
                let name = self.field_tokens.items[(usize)field]
                let rep = self.field_reps.items[(usize)field]
                if rep_array(rep) and rep_payload(rep) != 0:
                    # Decoded below; the field keeps its place in the spec.
                    try layer.set(spec, name, Value.marker(ValueKind.packed))
                    try self.arrays.push(PendingArray(spec = spec, field = name, rep = rep, value = Value.marker(ValueKind.empty), failed = false, failure = errors.invalid))
                else:
                    try layer.set(spec, name, try self.value(rep))
                at += 1
        if self.arrays.count > 0:
            try parallel.for_each(self.arrays.count, (void*)&self, decode_pending)
            for index in 0..<self.arrays.count:
                let pending = &self.arrays.items[index]
                if pending.failed:
                    if pending.failure == memory.exhausted: error(memory.exhausted, "out of memory decoding a crate array")
                    error(pending.failure, "a damaged array value in a crate file")
                try layer.set(pending.spec, pending.field, pending.value)
                pending.value = Value.marker(ValueKind.empty)

## Decode one pending array on the pool.
func decode_pending(context: void*, index: usize, worker: usize):
    let raw = context
    let crate = (Crate*)raw
    let pending = &crate.arrays.items[index]
    pending.value = crate.array(pending.rep) catch failure:
        pending.failed = true
        pending.failure = failure.code
        recover Value.marker(ValueKind.empty)

## ValueRep fields.
inline func rep_array(rep: u64) -> bool:
    return (rep >> 63) & 1 != 0

inline func rep_inlined(rep: u64) -> bool:
    return (rep >> 62) & 1 != 0

inline func rep_compressed(rep: u64) -> bool:
    return (rep >> 61) & 1 != 0

inline func rep_array_edit(rep: u64) -> bool:
    return (rep >> 60) & 1 != 0

inline func rep_type(rep: u64) -> u8:
    return (u8)((rep >> 48) & 0xFF)

inline func rep_payload(rep: u64) -> u64:
    return rep & 0xFFFFFFFFFFFF