Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-fbx

Source of release 0.12.1

0.12.1 / src / document / arrays.lucb

## Array decoding in one parallel batch. The converter plans every array a
## load needs (an `ArrayJob`: a node's array, the element type wanted and
## exactly sized output) and hands them over together; the batch runs on
## luce-geocore's pool:
##
## - a zlib array inflates whole, as one item, straight into its output when
##   the file's element type is the one wanted (else into scratch, then
##   converted);
## - a raw array is copied or converted in slices;
## - an ASCII array's text is cut at separators into pieces, which count
##   their numbers in parallel, then parse them in parallel into their place.
##
## A node's array is its first property when that is an array; otherwise its
## numeric scalar properties (short FBX 6 lists and single values).

## The element types the converter asks for.
pub enum Element as u8:
    real32 = 0
    real64 = 1
    int32 = 2
    int64 = 3
    byte = 4

pub func element_size(element: Element) -> usize:
    match element:
        .real32, .int32: return 4
        .real64, .int64: return 8
        _: return 1

## One array to decode: the array of `node` into `out`, which holds exactly
## its elements as `element`s.
pub struct ArrayJob:
    pub var node: u32
    pub var element: Element
    pub var out: u8[]

## Raw elements per item.
let raw_slice: usize = 1048576
## ASCII array text per piece, at least.
let text_piece: usize = 262144

## One item of the batch: part of one job.
struct Piece:
    var job: usize
    ## Source bytes (raw: element range; text: byte range) and the first
    ## element this piece writes.
    var start: usize
    var end: usize
    var first: usize
    var count: usize
    var failed: bool
    ## Why it failed.
    var code: ErrorCode
    var message: str

struct Batch:
    var document: const Document*
    var jobs: const ArrayJob[]
    var pieces: Piece[]

extend Document:
    ## The array property of `node`, if its first property is one.
    pub func array_property(node: u32) -> const Property*?:
        let found = self.property(node, 0) else return none
        if found.kind == 'b' or found.kind == 'i' or found.kind == 'l' or found.kind == 'f' or found.kind == 'd' or found.kind == 'a': return found
        return none

    ## How many elements `node`'s array holds.
    pub func array_count(node: u32) -> usize:
        if let found = self.array_property(node): return found.count
        var count: usize = 0
        for index in 0..<self.property_count(node):
            let found = self.property(node, index) else break
            if is_numeric_scalar(found.kind): count += 1
        return count

    ## Decode `node`'s array into a fresh heap span of `element`s (small
    ## arrays: settings, matrices, keys); the caller frees it.
    pub func decoded[T](node: u32, element: Element) -> T[]!:
        let count = self.array_count(node)
        let out = try new T[count] --- in memory.heap
        errdefer free(out) in memory.heap
        let bytes = u8[]((u8*)out.data, count * memory.size_of(T))
        let jobs: ArrayJob[1] = [ArrayJob(node, element, bytes)]
        try self.decode_arrays(jobs[..<1])
        return out

    ## Decode every job, in parallel.
    pub func decode_arrays(jobs: const ArrayJob[]) -> !:
        for job in jobs:
            if job.out.length != self.array_count(job.node) * element_size(job.element):
                error(errors.invalid, "an FBX array decode was planned with the wrong size")
        var pieces = Growing[Piece]()
        defer pieces.close()
        try self.plan(jobs, &pieces)
        var batch = Batch(&self, jobs, pieces.items[..<pieces.count])
        # Text pieces count their numbers first, to know where they write.
        var counting = false
        for piece in batch.pieces:
            if let found = self.array_property(jobs[piece.job].node):
                if found.kind == 'a': counting = true
        if counting:
            pool.run(batch.pieces.length, count_piece, (void*)&batch)
            try self.place_text_pieces(&batch)
        pool.run(batch.pieces.length, decode_piece, (void*)&batch)
        for piece in batch.pieces:
            if piece.failed: error(piece.code, piece.message)

    ## The items for `jobs`: zlib arrays whole and first (the longest work
    ## starts earliest), raw arrays in slices, text in pieces.
    func plan(jobs: const ArrayJob[], pieces: Growing[Piece]*) -> !:
        for pass in 0..<2:
            for (index, job) in jobs.indexed():
                var encoding = Encoding.raw
                var whole = true
                if let found = self.array_property(job.node):
                    encoding = found.encoding
                    whole = encoding == Encoding.zlib
                if (pass == 0) != (encoding == Encoding.zlib): continue
                let count = self.array_count(job.node)
                if count == 0: continue
                if whole:
                    try pieces.push(Piece(job = index, start = 0, end = count, first = 0, count = count, failed = false, code = errors.invalid, message = ""))
                elif encoding == Encoding.raw:
                    var first: usize = 0
                    while first < count:
                        let end = first + raw_slice if count - first > raw_slice else count
                        try pieces.push(Piece(job = index, start = first, end = end, first = first, count = end - first, failed = false, code = errors.invalid, message = ""))
                        first = end
                else:
                    # Text: cut after a separator every text_piece bytes.
                    let found = self.array_property(job.node) else continue
                    let text = self.data[found.offset..<found.offset + found.size]
                    var start: usize = 0
                    while start < text.length:
                        var end = start + text_piece
                        if end >= text.length:
                            end = text.length
                        else:
                            while end < text.length and text[end] != ',': end += 1
                            if end < text.length: end += 1
                        try pieces.push(Piece(job = index, start = start, end = end, first = 0, count = 0, failed = false, code = errors.invalid, message = ""))
                        start = end

    ## After counting: each text piece's first element, and each array's
    ## total checked against its declared count.
    func place_text_pieces(batch: Batch*) -> !:
        var at: usize = 0
        while at < batch.pieces.length:
            let job = batch.pieces[at].job
            let property = self.array_property(batch.jobs[job].node)
            if property == none:
                at += 1
                continue
            let found = property else return
            if found.kind != 'a':
                at += 1
                continue
            var total: usize = 0
            while at < batch.pieces.length and batch.pieces[at].job == job:
                if batch.pieces[at].failed: error(errors.invalid, "an ASCII FBX array holds something other than numbers")
                batch.pieces[at].first = total
                total += batch.pieces[at].count
                at += 1
            if total != found.count: error(errors.invalid, "an ASCII FBX array holds a different number of values than it declares")

func is_numeric_scalar(kind: u8) -> bool:
    return kind == 'C' or kind == 'Y' or kind == 'I' or kind == 'L' or kind == 'F' or kind == 'D' or kind == 'N'

func count_piece(context: void*?, index: usize):
    let batch = (Batch*)(context else return)
    let piece = &batch.pieces[index]
    let found = batch.document.array_property(batch.jobs[piece.job].node) else return
    if found.kind != 'a': return
    let text = batch.document.data[found.offset + piece.start..<found.offset + piece.end]
    if let count = count_numbers(text):
        piece.count = count
    else:
        piece.failed = true

## How many numbers a piece of array text holds, or none when something
## other than numbers, separators and white space is there.
func count_numbers(text: const u8[]) -> usize?:
    var count: usize = 0
    var inside = false
    var comment = false
    for byte in text:
        if comment:
            if byte == '\n': comment = false
        elif byte == ';':
            comment = true
            inside = false
        elif byte == ',' or byte == ' ' or byte == '\n' or byte == '\r' or byte == '\t':
            inside = false
        elif is_number_byte(byte):
            if not inside: count += 1
            inside = true
        else:
            return none
    return count

func is_number_byte(byte: u8) -> bool:
    return (byte >= '0' and byte <= '9') or byte == '-' or byte == '+' or byte == '.' or byte == '#' or byte == '(' or byte == ')' or byte == '_' or (byte >= 'A' and byte <= 'Z') or (byte >= 'a' and byte <= 'z')

func decode_piece(context: void*?, index: usize):
    let batch = (Batch*)(context else return)
    let piece = &batch.pieces[index]
    decode_one(batch, *piece) catch failure:
        piece.failed = true
        piece.code = failure.code
        piece.message = failure.message

## Decode one piece into its job's output.
func decode_one(batch: const Batch*, piece: Piece) -> !:
    let document = batch.document
    let job = batch.jobs[piece.job]
    let size = element_size(job.element)
    let out = job.out[piece.first * size..<(piece.first + piece.count) * size]
    let property = document.array_property(job.node)
    if property == none:
        try scalars(document, job.node, job.element, out)
        return
    let found = property else return
    if found.encoding == Encoding.text:
        let text = document.data[found.offset + piece.start..<found.offset + piece.end]
        try parse_numbers(text, job.element, out, piece.count)
        return
    let source_size = element_bytes(found.kind)
    if found.encoding == Encoding.raw and not document.big:
        let bytes = document.data[found.offset + piece.start * source_size..<found.offset + piece.end * source_size]
        convert(found.kind, bytes, job.element, out, piece.count)
        return
    if found.encoding == Encoding.zlib and same_layout(found.kind, job.element) and not document.big:
        try compress.inflate(document.data[found.offset..<found.offset + found.size], out)
        return
    # Through scratch: a zlib array of another type, or a big-endian array
    # (its elements swapped first).
    let scratch = try new u8[piece.count * source_size] --- in memory.heap
    defer free(scratch) in memory.heap
    if found.encoding == Encoding.raw:
        memory.copy(scratch, document.data[found.offset + piece.start * source_size..], scratch.length)
    else:
        try compress.inflate(document.data[found.offset..<found.offset + found.size], scratch)
    if document.big: swap_elements(scratch, source_size)
    convert(found.kind, scratch, job.element, out, piece.count)

## Reverse the bytes of each `size`-byte element.
func swap_elements(bytes: u8[], size: usize):
    if size < 2: return
    var at: usize = 0
    while at + size <= bytes.length:
        for index in 0..<size // 2:
            let byte = bytes[at + index]
            bytes[at + index] = bytes[at + size - 1 - index]
            bytes[at + size - 1 - index] = byte
        at += size

## Whether an array of `kind` already has `element`'s bytes.
func same_layout(kind: u8, element: Element) -> bool:
    match element:
        .real32: return kind == 'f'
        .real64: return kind == 'd'
        .int32: return kind == 'i'
        .int64: return kind == 'l'
        _: return kind == 'b'

## `count` elements of binary array type `kind` in `source` as `element`s.
func convert(kind: u8, source: const u8[], element: Element, out: u8[], count: usize):
    if same_layout(kind, element):
        memory.copy(out, source, count * element_size(element))
        return
    # The common narrowing and widening in plain loops (the file's values
    # may be unaligned; the output is not).
    if kind == 'd' and element == Element.real32:
        let into = (f32*)out.data
        for index in 0..<count: into[index] = (f32)memory.read[f64]((void*)&source[index * 8])
        return
    if kind == 'f' and element == Element.real64:
        let into = (f64*)out.data
        for index in 0..<count: into[index] = (f64)memory.read[f32]((void*)&source[index * 4])
        return
    for index in 0..<count:
        match kind:
            'd': put_real(out, index, element, memory.read[f64]((void*)&source[index * 8]))
            'f': put_real(out, index, element, (f64)memory.read[f32]((void*)&source[index * 4]))
            'i': put_integer(out, index, element, (i64)memory.read[i32]((void*)&source[index * 4]))
            'l': put_integer(out, index, element, memory.read[i64]((void*)&source[index * 8]))
            _: put_integer(out, index, element, (i64)source[index])

func put_real(out: u8[], index: usize, element: Element, value: f64):
    match element:
        .real32: memory.write[f32]((void*)&out[index * 4], (f32)value)
        .real64: memory.write[f64]((void*)&out[index * 8], value)
        .int32: memory.write[i32]((void*)&out[index * 4], (i32)value)
        .int64: memory.write[i64]((void*)&out[index * 8], (i64)value)
        _: out[index] = (u8)(value != 0.0)

func put_integer(out: u8[], index: usize, element: Element, value: i64):
    match element:
        .real32: memory.write[f32]((void*)&out[index * 4], (f32)value)
        .real64: memory.write[f64]((void*)&out[index * 8], (f64)value)
        .int32: memory.write[i32]((void*)&out[index * 4], (i32)value)
        .int64: memory.write[i64]((void*)&out[index * 8], value)
        _: out[index] = (u8)(value != 0)

## Parse the numbers of array text into `count` elements.
func parse_numbers(text: const u8[], element: Element, out: u8[], count: usize) -> !:
    var at: usize = 0
    var index: usize = 0
    while at < text.length:
        let byte = text[at]
        if byte == ';':
            while at < text.length and text[at] != '\n': at += 1
            continue
        if not is_number_byte(byte):
            at += 1
            continue
        let start = at
        while at < text.length and is_number_byte(text[at]): at += 1
        if index >= count: error(errors.invalid, "an ASCII FBX array holds more values than it declares")
        let token = text[start..<at]
        if element == Element.real32 or element == Element.real64:
            put_real(out, index, element, parse_real(token))
        elif let integer = parse_integer(token):
            put_integer(out, index, element, integer)
        else:
            put_real(out, index, element, parse_real(token))
        index += 1
    if index != count: error(errors.invalid, "an ASCII FBX array holds fewer values than it declares")

## A node's numeric scalar properties as an array.
func scalars(document: const Document*, node: u32, element: Element, out: u8[]) -> !:
    var index: usize = 0
    for at in 0..<document.property_count(node):
        let found = document.property(node, at) else break
        if not is_numeric_scalar(found.kind): continue
        if found.kind == 'F' or found.kind == 'D' or (found.kind == 'N' and (element == Element.real32 or element == Element.real64)):
            put_real(out, index, element, found.real)
        else:
            put_integer(out, index, element, found.integer)
        index += 1