Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-fbx

Source of release 0.12.1

0.12.1 / src / writer / encode.lucb

## A Tree as binary FBX, laid out as Blender's encoder (encode_bin.py) and
## the FBX SDK write it:
##
## - a 27-byte header: "Kaydara FBX Binary  \0", 0x1a 0x00, the version;
## - node records: end offset, property count and property bytes (u32
##   each in 7400, u64 in 7500), the name's length and bytes, the
##   properties, the children, and a null record after children (or after
##   a node with neither properties nor children);
## - a null record closing the top level, then the footer (Blender's fixed
##   FileId/CreationTime pair and footer id, padding to 16, the version,
##   120 zero bytes and the closing magic).
##
## Arrays over 128 bytes are deflated first (zlib level 1, encoding 1), one
## pool task each, a large one in 1 MiB segments on several threads (what
## compress.compress does); smaller ones are stored raw (encoding 0), as
## Blender does.

let magic: u8[23] = [0x4b, 0x61, 0x79, 0x64, 0x61, 0x72, 0x61, 0x20, 0x46, 0x42, 0x58, 0x20, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x20, 0x20, 0x00, 0x1a, 0x00]
pub let file_id: u8[16] = [0x28, 0xb3, 0x2a, 0xeb, 0xb6, 0x24, 0xcc, 0xc2, 0xbf, 0xc8, 0xb0, 0x2a, 0xa9, 0x2b, 0xfc, 0xf1]
pub let creation_time: str = "1970-01-01 10:00:00:000"
let foot_id: u8[16] = [0xfa, 0xbc, 0xab, 0x09, 0xd0, 0xc8, 0xd4, 0x66, 0xb1, 0x76, 0xfb, 0x83, 0x1c, 0xf7, 0x26, 0x7e]
let foot_magic: u8[16] = [0xf8, 0x5a, 0x8c, 0x6a, 0xde, 0xf5, 0xd9, 0x7e, 0xec, 0xe9, 0x0c, 0xe3, 0x75, 0x8f, 0x29, 0x0b]

## Arrays of more bytes than this are deflated.
let pack_above: usize = 128

## What an encode does.
struct Encoding:
    ## 7400, or 7500 (64-bit records); 0 picks 7400 unless the file needs
    ## 7500's offsets.
    var version: u32
    var deflate: bool

struct PackWork:
    var blobs: Blob[]

## Deflate blob `index` (zlib level 1) when it is large enough;
## compress.compress takes a large one in 1 MiB segments on several threads,
## which compress.inflate reads back the same way.
func pack_blob(context: void*?, index: usize):
    let work = (PackWork*)(context else return)
    let blob = &work.blobs[index]
    if blob.raw.length <= pack_above: return
    blob.packed = compress.compress(blob.raw, 1) catch failure:
        _ = failure
        blob.failed = true
        recover empty_bytes()
    blob.packed_length = blob.packed.length

## Deflate every array over `pack_above` bytes, one pool task each.
func pack_blobs(tree: Tree*) -> !:
    var work = PackWork(blobs = tree.blobs.items[..<tree.blobs.count])
    pool.run(tree.blobs.count, pack_blob, (void*)&work)
    for blob in tree.blobs.view():
        if blob.failed: error(memory.exhausted, "out of memory deflating an FBX array")

## `version_prop` is the header's FBXVersion property, set to the version
## written.
func encode_tree(tree: Tree*, requested: u32, deflate: bool, version_prop: usize) -> u8[]!:
    if deflate: try pack_blobs(tree)
    var version = requested
    var sizes = try new u64[tree.nodes.count] --- in memory.heap
    defer free(sizes) in memory.heap
    var total = measure(tree, sizes, false)
    if version == 0:
        version = 7400
        if total > 4294967295: version = 7500
    if version >= 7500: total = measure(tree, sizes, true)
    elif total > 4294967295: error(errors.limit, "the FBX file needs 64-bit offsets: write version 7500")
    tree.props.items[version_prop].bits = (u64)version
    let data = try new u8[(usize)total] --- in memory.heap
    errdefer free(data) in memory.heap
    var out = Output(data = data, at = 0, wide = version >= 7500, copies = Growing[Copy]())
    defer out.copies.close()
    for byte in magic: out.put8(byte)
    out.put32(version)
    for root in tree.roots.view(): try write_node(tree, sizes, root, &out)
    # The arrays' bytes, copied in parallel into the places left for them.
    var copying = CopyWork(tree = tree, data = data, copies = out.copies.view())
    pool.run(out.copies.count, copy_array, (void*)&copying)
    out.zeros(header_bytes(out.wide))
    for byte in foot_id: out.put8(byte)
    out.zeros(4)
    out.zeros(padding(out.at))
    out.put32(version)
    out.zeros(120)
    for byte in foot_magic: out.put8(byte)
    if out.at != data.length: error(errors.invalid, "the FBX encoder's layout does not add up")
    return data

## A record header's bytes (the null record's too).
func header_bytes(wide: bool) -> usize:
    return 25 if wide else 13

## Footer padding after the footer id and its four zeros: to a multiple of
## 16, 16 when already aligned.
func padding(at: u64) -> usize:
    let gap = (usize)(((at + 15) & ~(u64)15) - at)
    return 16 if gap == 0 else gap

## Bytes of one property.
func prop_bytes(tree: const Tree*, prop: const WProp*) -> u64:
    match prop.kind:
        'C', 'B': return 2
        'Y': return 3
        'I', 'F': return 5
        'L', 'D': return 9
        'S', 'R': return 5 + (u64)prop.length
        _:
            let blob = &tree.blobs.items[prop.start]
            return 13 + (u64)(blob.packed_length if blob.packed_length > 0 else blob.raw.length)

func props_bytes(tree: const Tree*, node: const WNode*) -> u64:
    var total: u64 = 0
    for at in node.first_prop..<node.first_prop + node.prop_count:
        total += prop_bytes(tree, &tree.props.items[(usize)at])
    return total

## Whether a node ends with a null record: it has children, or nothing.
func closes_with_null(node: const WNode*, index: u32) -> bool:
    return node.end > index + 1 or node.prop_count == 0

## Every node's record size (last first, so children are known), and the
## whole file's size.
func measure(tree: const Tree*, sizes: u64[], wide: bool) -> u64:
    let header = (u64)header_bytes(wide)
    var at = tree.nodes.count
    while at > 0:
        at -= 1
        let node = &tree.nodes.items[at]
        var size = header + (u64)node.name_length + props_bytes(tree, node)
        var child = (u32)at + 1
        while child < node.end:
            size += sizes[(usize)child]
            child = tree.nodes.items[(usize)child].end
        if closes_with_null(node, (u32)at): size += header
        sizes[at] = size
    var total: u64 = 27
    for root in tree.roots.view(): total += sizes[(usize)root]
    total += header + 20
    total += (u64)padding(total) + 4 + 120 + 16
    return total

## A cursor over the output.
struct Output:
    var data: u8[]
    var at: u64
    var wide: bool
    ## Array bytes still to copy: where, from which blob, and which piece.
    var copies: Growing[Copy]

    func put8(value: u8):
        self.data[(usize)self.at] = value
        self.at += 1

    func put32(value: u32):
        memory.write[u32]((void*)&self.data[(usize)self.at], value)
        self.at += 4

    func put64(value: u64):
        memory.write[u64]((void*)&self.data[(usize)self.at], value)
        self.at += 8

    ## A record-header number: u64 in 7500, u32 before.
    func number(value: u64):
        if self.wide: self.put64(value)
        else: self.put32((u32)value)

    func bytes(value: const u8[]):
        if value.length > 0: memory.copy(self.data[(usize)self.at..], value, value.length)
        self.at += (u64)value.length

    func zeros(count: usize):
        for k in 0..<count: self.data[(usize)self.at + k] = 0
        self.at += (u64)count

## Node `index` and its subtree.
func write_node(tree: const Tree*, sizes: const u64[], index: u32, out: Output*) -> !:
    let node = &tree.nodes.items[(usize)index]
    let end = out.at + sizes[(usize)index]
    out.number(end)
    out.number((u64)node.prop_count)
    out.number(props_bytes(tree, node))
    out.put8((u8)node.name_length)
    out.bytes(tree.text.items[(usize)node.name_start..<(usize)(node.name_start + node.name_length)])
    for at in node.first_prop..<node.first_prop + node.prop_count:
        try write_prop(tree, &tree.props.items[(usize)at], out)
    var child = index + 1
    while child < node.end:
        try write_node(tree, sizes, child, out)
        child = tree.nodes.items[(usize)child].end
    if closes_with_null(node, index): out.zeros(header_bytes(out.wide))

func write_prop(tree: const Tree*, prop: const WProp*, out: Output*) -> !:
    out.put8(prop.kind)
    match prop.kind:
        'C', 'B': out.put8((u8)prop.bits)
        'Y':
            out.put8((u8)prop.bits)
            out.put8((u8)(prop.bits >> 8))
        'I', 'F': out.put32((u32)prop.bits)
        'L', 'D': out.put64(prop.bits)
        'S', 'R':
            out.put32((u32)prop.length)
            out.bytes(tree.text.items[prop.start..<prop.start + prop.length])
        _:
            let blob = &tree.blobs.items[prop.start]
            let packed = blob.packed_length > 0
            out.put32((u32)blob.count)
            out.put32(1 if packed else 0)
            out.put32((u32)(blob.packed_length if packed else blob.raw.length))
            # The bytes come later (encode_tree), a megabyte-sized piece a task.
            let length = blob.packed_length if packed else blob.raw.length
            var start: usize = 0
            while start < length:
                let end = start + copy_piece if length - start > copy_piece else length
                try out.copies.push(Copy(at = (usize)out.at + start, blob = prop.start, start = start, end = end))
                start = end
            out.at += (u64)length

## Array bytes copied in pieces of at most this many.
let copy_piece: usize = 1048576

## One piece of an array's bytes and where it goes.
struct Copy:
    var at: usize
    var blob: usize
    var start: usize
    var end: usize

struct CopyWork:
    var tree: const Tree*
    var data: u8[]
    var copies: const Copy[]

func copy_array(context: void*?, index: usize):
    let work = (CopyWork*)(context else return)
    let piece = work.copies[index]
    let blob = &work.tree.blobs.items[piece.blob]
    let bytes = blob.packed if blob.packed_length > 0 else blob.raw
    memory.copy(work.data[piece.at..], bytes[piece.start..<piece.end], piece.end - piece.start)