dymokomi/luce-prism
Source of release 0.6.1
0.6.1 / src / serialization / binary.lucb
## Serialization binary.
import memory
from animation import Sample
from model import DocumentData
from path import require_element, require_slot
from serialization.compression import compress_bytes, decompress_bytes
from storage import Buffer, List, Reader, add_name, set_pair, text_copy
from types import Interpolation, invalid, limit_exceeded, unsupported
from value import read_value, write_value
from names import Name
## The writer's string table: each distinct string once, found through a hash index
## (a linear search made writing a snapshot quadratic in its element count).
pub struct StringTable:
pub var strings: List[str]
## Open addressing: 0 is empty, otherwise a string index plus one.
var slots: usize[]
pub func init():
return
pub func close():
self.strings.close()
if self.slots.length > 0: free(self.slots) in memory.heap
func slot_for(text: str) -> usize:
var value: u64 = 14695981039346656037
for byte in text.bytes: value = (value ^ (u64)byte) *% 1099511628211
let mask = self.slots.length - 1
var at = (usize)value & mask
while self.slots[at] != 0 and self.strings.items[self.slots[at] - 1] != text:
at = (at + 1) & mask
return at
func grow() -> !:
var capacity: usize = 64
while capacity < (self.strings.count + 1) * 2: capacity *= 2
let slots = try new usize[capacity] --- in memory.heap
for i in 0..<capacity: slots[i] = 0
if self.slots.length > 0: free(self.slots) in memory.heap
self.slots = slots
for i in 0..<self.strings.count: self.slots[self.slot_for(self.strings.items[i])] = i + 1
pub func intern(text: str) -> usize!:
if (self.strings.count + 1) * 2 > self.slots.length: try self.grow()
let at = self.slot_for(text)
if self.slots[at] != 0: return self.slots[at] - 1
try self.strings.push(text)
self.slots[at] = self.strings.count
return self.strings.count - 1
pub func index(text: str) -> usize:
if self.slots.length > 0:
let at = self.slot_for(text)
if self.slots[at] != 0: return self.slots[at] - 1
trap("Prism writer missed an interned string")
pub func intern(table: StringTable*, text: str) -> usize!:
return try table.intern(text)
pub func string_index(table: const StringTable*, text: str) -> usize:
return table.index(text)
pub func write_index(out: Buffer*, table: const StringTable*, text: str) -> !:
try out.uint((u64)string_index(table, text), 4)
pub func read_index(reader: Reader*, table: const List[str]*) -> str!:
let index = (usize)try reader.uint(4)
if index >= table.count:
error(invalid, "invalid Prism string-table index")
let text = table.items[index]
if text.length > reader.budget:
error(limit_exceeded, "Prism expanded string budget exceeded")
reader.budget -= text.length
return text
pub func intern_names(table: StringTable*, names: const List[Buffer]*) -> !:
for name in names.span():
_ = try intern(table, name.view())
pub func write_names(out: Buffer*, table: const StringTable*, names: const List[Buffer]*) -> !:
try out.uint((u64)names.count, 4)
for name in names.span():
try write_index(out, table, name.view())
pub func read_names(reader: Reader*, table: const List[str]*, names: List[Buffer]*) -> !:
let count = try reader.count(4)
for i in 0..<count:
let name = try read_index(reader, table)
let before = names.count
try add_name(names, name)
if names.count == before:
error(invalid, "duplicate Prism tombstone")
pub func write_binary(document: const DocumentData*, compressed: bool = false) -> Buffer!:
var table = StringTable()
defer table.close()
for element in document.elements.span():
_ = try intern(&table, element.path.view())
_ = try intern(&table, element.kind.view())
for prop in element.properties.span():
_ = try intern(&table, prop.name.view())
for meta in element.metadata.span():
_ = try intern(&table, meta.key.view())
_ = try intern(&table, meta.value.view())
try intern_names(&table, &element.removed_properties)
try intern_names(&table, &element.removed_metadata)
for connection in document.connections.span():
_ = try intern(&table, connection.value.view())
_ = try intern(&table, connection.key.view())
try intern_names(&table, &document.deletions)
try intern_names(&table, &document.disconnections)
for order in document.reorders.span():
_ = try intern(&table, order.parent.view())
try intern_names(&table, &order.children)
for glob in document.globs.span():
_ = try intern(&table, glob.key.view())
_ = try intern(&table, glob.value.view())
var body: Buffer
defer body.close()
try body.uint((u64)table.strings.count, 4)
for text in table.strings.span():
try body.sized(text)
try body.uint((u64)document.elements.count, 4)
for element in document.elements.span():
try write_index(&body, &table, element.path.view())
try write_index(&body, &table, element.kind.view())
try body.uint((u64)element.properties.count, 4)
for prop in element.properties.span():
try write_index(&body, &table, prop.name.view())
try write_value(&body, &prop.value)
try body.byte(1 if prop.samples.count > 0 else 0)
if prop.samples.count > 0:
try body.uint((u64)prop.samples.count, 4)
for sample in prop.samples.span():
try body.real(sample.time)
try write_value(&body, &sample.value)
try body.uint((u64)sample.interpolation, 1)
try body.byte(sample.handle)
try body.uint((u64)(sample.left.count // 2), 4)
for value in sample.left.span():
try body.real(value)
try body.uint((u64)(sample.right.count // 2), 4)
for value in sample.right.span():
try body.real(value)
try body.sized(prop.type_name.view())
try body.uint((u64)element.metadata.count, 4)
for meta in element.metadata.span():
try write_index(&body, &table, meta.key.view())
try write_index(&body, &table, meta.value.view())
try body.byte(1 if element.override else 0)
try write_names(&body, &table, &element.removed_properties)
try write_names(&body, &table, &element.removed_metadata)
try body.uint((u64)document.connections.count, 4)
for connection in document.connections.span():
try write_index(&body, &table, connection.value.view())
try write_index(&body, &table, connection.key.view())
try write_names(&body, &table, &document.deletions)
try write_names(&body, &table, &document.disconnections)
if document.reorders.count > 0:
try body.byte(1)
try body.uint((u64)document.reorders.count, 4)
for order in document.reorders.span():
try write_index(&body, &table, order.parent.view())
try write_names(&body, &table, &order.children)
if document.globs.count > 0:
try body.byte(2)
try body.uint((u64)document.globs.count, 4)
for glob in document.globs.span():
try write_index(&body, &table, glob.key.view())
try write_index(&body, &table, glob.value.view())
var output: Buffer
errdefer output.close()
try output.text("PRSMC\0")
try output.uint(4, 2)
try output.uint(1 if compressed else 0, 2)
if compressed:
var packed = try compress_bytes(body.bytes())
defer packed.close()
try output.put(packed.bytes())
else:
try output.put(body.bytes())
return output
pub func read_binary(bytes: const u8[], document: DocumentData*) -> !:
var header = Reader(data = bytes, pos = 6)
if try header.uint(2) != 4:
error(unsupported, "only Prism v4 binary crates are supported")
let flags = try header.uint(2)
if flags > 1:
error(unsupported, "unknown Prism crate flags")
var inflated: Buffer
defer inflated.close()
var payload = bytes[header.pos..]
if flags == 1:
inflated = try decompress_bytes(payload)
payload = inflated.bytes()
var reader = Reader(data = payload)
var table: List[str]
defer table.close()
let string_count = try reader.count(4)
for i in 0..<string_count:
try table.push(try reader.text())
let element_count = try reader.count(25)
for i in 0..<element_count:
let path = try read_index(&reader, &table)
let kind = try read_index(&reader, &table)
try document.insert(path, kind)
let property_count = try reader.count(8)
for p in 0..<property_count:
let name = try read_index(&reader, &table)
if document.elements.items[i].property(name) != none:
error(invalid, "duplicate Prism property")
var value = try read_value(&reader)
try document.put_value(path, name, value)
let prop_index = document.elements.items[i].property(name) else trap("inserted Prism property missing")
let prop = &document.elements.items[i].properties.items[prop_index]
let animated = try reader.uint(1)
if animated > 1:
error(invalid, "invalid Prism animation flag")
if animated == 1:
let count = try reader.count(21)
for s in 0..<count:
var sample = Sample()
errdefer sample.close()
sample.time = try reader.real()
sample.value = try read_value(&reader)
let interpolation = try reader.uint(1)
if interpolation > 2:
error(invalid, "invalid Prism interpolation")
sample.interpolation = Interpolation.held if interpolation == 0 else (Interpolation.linear if interpolation == 1 else Interpolation.bezier)
sample.handle = (u8)try reader.uint(1)
if sample.handle > 3:
error(invalid, "invalid Prism handle type")
let left = try reader.count(16)
for h in 0..<left * 2:
try sample.left.push(try reader.real())
let right = try reader.count(16)
for h in 0..<right * 2:
try sample.right.push(try reader.real())
if s > 0 and sample.time <= prop.samples.items[s - 1].time:
error(invalid, "Prism keyframes must have increasing times")
try prop.samples.push(sample)
prop.type_name = try Name.of(try reader.text())
let metadata_count = try reader.count(8)
for m in 0..<metadata_count:
let key = try read_index(&reader, &table)
let value = try read_index(&reader, &table)
let before = document.elements.items[i].metadata.count
try document.set_metadata(path, key, value)
if before == document.elements.items[i].metadata.count:
error(invalid, "duplicate Prism metadata")
let element = &document.elements.items[i]
let override = try reader.uint(1)
if override > 1:
error(invalid, "invalid Prism override flag")
element.override = override == 1
try read_names(&reader, &table, &element.removed_properties)
try read_names(&reader, &table, &element.removed_metadata)
let connection_count = try reader.count(8)
for i in 0..<connection_count:
let source = try read_index(&reader, &table)
let target = try read_index(&reader, &table)
try require_slot(source)
try require_slot(target)
# Layers can refer to elements supplied by a base or reference.
let before = document.connections.count
try set_pair(&document.connections, target, source)
if document.connections.count == before:
error(invalid, "duplicate Prism connection target")
try read_names(&reader, &table, &document.deletions)
for path in document.deletions.span():
try require_element(path.view())
try read_names(&reader, &table, &document.disconnections)
for path in document.disconnections.span():
try require_slot(path.view())
var seen_reorders = false
var seen_globs = false
while reader.remaining() > 0:
let tag = try reader.uint(1)
if tag == 1 and not seen_reorders:
seen_reorders = true
let count = try reader.count(8)
for i in 0..<count:
let parent = try read_index(&reader, &table)
var children: List[str]
defer children.close()
let size = try reader.count(4)
for j in 0..<size:
try children.push(try read_index(&reader, &table))
let before = document.reorders.count
try document.reorder(parent, children.span())
if document.reorders.count == before:
error(invalid, "duplicate Prism reorder parent")
elif tag == 2 and not seen_globs:
seen_globs = true
let count = try reader.count(8)
for i in 0..<count:
let source = try read_index(&reader, &table)
let target = try read_index(&reader, &table)
try document.connect_pattern(source, target)
else:
error(unsupported, "unknown or duplicate Prism crate section")
## Legacy LZSS stream: u64 raw length, then LSB-first groups of eight tokens.
## Match = u16(distance-1), u8(length-4); overlapping references are legal.