dymokomi/luce-prism
Source of release 0.6.0
## Value.
import memory
import interop
from luce_std import math
from storage import Buffer, List, Reader, copied
from types import DType, invalid, limit_exceeded, max_bytes
pub func dtype_width(dtype: DType) -> usize:
if dtype == DType.string:
return 0
if dtype == DType.int16 or dtype == DType.uint16 or dtype == DType.float16:
return 2
if dtype == DType.int32 or dtype == DType.uint32 or dtype == DType.float32:
return 4
if dtype == DType.int64 or dtype == DType.uint64 or dtype == DType.float64:
return 8
return 1
pub func dtype_from(code: u64) -> DType!:
let types: DType[14] = [DType.boolean, DType.character, DType.int8, DType.int16, DType.int32, DType.int64, DType.uint8, DType.uint16, DType.uint32, DType.uint64, DType.float32, DType.float64, DType.string, DType.float16]
if code >= 14:
error(invalid, "unknown Prism dtype")
return types[(usize)code]
pub func floating(dtype: DType) -> bool:
return dtype == DType.float16 or dtype == DType.float32 or dtype == DType.float64
pub func shape_count(shape: const u32[]) -> usize!:
if shape.length > 16:
error(limit_exceeded, "Prism values support at most 16 dimensions")
var count: usize = 1
for dimension in shape:
count = count *? (usize)dimension else error(limit_exceeded, "Prism shape overflow")
if count > max_bytes:
error(limit_exceeded, "Prism value element limit exceeded")
return count
pub struct Locator:
pub var identity: Buffer
pub var generation: usize
pub var extent: usize
pub var size: usize
pub func init():
return
pub func copy() -> Locator!:
var out = Locator()
out.generation = self.generation
out.extent = self.extent
out.size = self.size
out.identity = try self.identity.copy()
return out
pub func close():
self.identity.close()
## `locator` on the heap, owned by the value that carries it.
pub func own_locator(locator: Locator) -> Locator*!:
var owned = locator
let out = new Locator() in memory.heap catch failure:
owned.close()
error(failure.code, failure.message)
out[0] = owned
return out
func drop_locator(locator: Locator*):
locator.close()
free(locator) in memory.heap
pub func is_stub(value: const ValueData*) -> bool:
let found = value.locator else return false
return value.payload.used == 0 and found.size > 0 and found.identity.used > 0
pub struct ValueData:
pub var kind: DType = DType.boolean
pub var shape: List[u32]
pub var payload: Buffer
## Where the payload was written when paged out; only such a value has one, so
## the rest do not carry its 48 bytes.
pub var locator: Locator*?
pub func count() -> usize:
var result: usize = 1
for dimension in self.shape.span():
result *= (usize)dimension
return result
pub func copy() -> ValueData!:
var out = try value_header(self.kind, self.shape.span())
errdefer out.close()
out.payload = try self.payload.copy()
if let found = self.locator:
out.locator = try own_locator(try found.copy())
return out
pub func bits(index: usize) -> u64:
let width = dtype_width(self.kind)
var bits: u64 = 0
for i in 0..<width:
bits |= (u64)self.payload.data[index * width + i] << (i * 8)
return bits
pub func number(index: usize) -> f64!:
if index >= self.count() or self.kind == DType.string:
error(invalid, "expected a numeric Prism component")
var bits = self.bits(index)
if self.kind == DType.boolean:
return 1.0 if bits != 0 else 0.0
if self.kind == DType.float64:
return memory.read[f64](&bits)
if self.kind == DType.float32:
var narrow = (u32)bits
return (f64)memory.read[f32](&narrow)
if self.kind == DType.float16:
return half_number((u16)bits)
if self.kind == DType.int8 or self.kind == DType.int16 or self.kind == DType.int32 or self.kind == DType.int64:
return (f64)self.signed(index)
return (f64)bits
pub func signed(index: usize) -> i64:
let width = dtype_width(self.kind)
var bits = self.bits(index)
if width < 8 and (bits & ((u64)1 << (width * 8 - 1))) != 0:
bits |= 18446744073709551615 << (width * 8)
return memory.read[i64](&bits)
pub func close():
self.shape.close()
self.payload.close()
if let found = self.locator:
drop_locator(found)
self.locator = none
pub func value_header(dtype: DType, shape: const u32[]) -> ValueData!:
_ = try shape_count(shape)
var out = ValueData(kind = dtype)
errdefer out.close()
for dimension in shape:
try out.shape.push(dimension)
return out
pub func read_value(reader: Reader*) -> ValueData!:
let dtype = try dtype_from(try reader.uint(1))
let rank = (usize)try reader.uint(1)
if rank > 16:
error(limit_exceeded, "Prism rank exceeds 16")
var shape: u32[16]
for i in 0..<rank:
shape[i] = (u32)try reader.uint(4)
var out = try value_header(dtype, shape[..<rank])
errdefer out.close()
let count = out.count()
if dtype == DType.string:
if count > reader.remaining() // 4:
error(invalid, "truncated Prism string array")
for i in 0..<count:
try out.payload.sized(try reader.text())
else:
let size = count *? dtype_width(dtype) else error(limit_exceeded, "Prism payload overflow")
out.payload = try copied(try reader.take(size))
return out
pub func write_value(out: Buffer*, value: const ValueData*) -> !:
try out.uint((u64)value.kind, 1)
try out.uint((u64)value.shape.count, 1)
for dimension in value.shape.span():
try out.uint((u64)dimension, 4)
# Preserve half bit patterns, as the C++ writer does. Other non-finite
# floating payloads normalize to zero in both legacy encodings.
if value.kind == DType.float32 or value.kind == DType.float64:
for i in 0..<value.count():
let finite = math.is_finite(try value.number(i))
try out.uint(value.bits(i) if finite else 0, dtype_width(value.kind))
else:
try out.put(value.payload.bytes())
pub func scalar_bits(dtype: DType, bits: u64) -> ValueData!:
var out = ValueData(kind = dtype)
errdefer out.close()
try out.payload.uint(bits, dtype_width(dtype))
return out
pub func half_number(bits: u16) -> f64:
let sign = -1.0 if (bits & 32768) != 0 else 1.0
let exponent = (bits >> 10) & 31
let mantissa = bits & 1023
if exponent == 31:
var raw: u64 = 9218868437227405312 if mantissa == 0 else 9221120237041090560
return sign * memory.read[f64](&raw)
var value = (f64)mantissa / 1024.0 if exponent == 0 else 1.0 + (f64)mantissa / 1024.0
var power = -14 if exponent == 0 else (i32)exponent - 15
while power < 0:
value *= 0.5
power += 1
while power > 0:
value *= 2.0
power -= 1
return sign * value
pub func half_bits(number: f64) -> u16:
var value = (f32)number
let bits = memory.read[u32](&value)
let sign = (bits >> 16) & 32768
var mantissa = bits & 8388607
let exponent = (i32)((bits >> 23) & 255) - 112
if ((bits >> 23) & 255) == 255:
return (u16)(sign | 31744 | (512 if mantissa != 0 else 0))
if exponent >= 31:
return (u16)(sign | 31744)
if exponent <= 0:
if exponent < -10:
return (u16)sign
mantissa |= 8388608
let shift = (u32)(14 - exponent)
var rounded = mantissa >> shift
if ((mantissa >> (shift - 1)) & 1) != 0:
rounded += 1
return (u16)(sign | rounded)
var result = sign | ((u32)exponent << 10) | (mantissa >> 13)
if (mantissa & 4096) != 0:
result += 1
return (u16)result
pub func append_number(payload: Buffer*, kind: DType, value: f64) -> !:
if kind == DType.float64:
var number = value
try payload.uint(memory.read[u64](&number), 8)
elif kind == DType.float32:
var number = (f32)value
try payload.uint((u64)memory.read[u32](&number), 4)
else:
try payload.uint((u64)half_bits(value), 2)
pub struct Value:
var data: ValueData
pub func view_data() -> const ValueData*:
return &self.data
pub func init(number: f64 = 0.0) -> !:
var value = number
self.data = try scalar_bits(DType.float64, memory.read[u64](&value))
pub static func integer(value: i64) -> interop.Reference[Value]!:
var signed_value = value
return try own_value(try scalar_bits(DType.int64, memory.read[u64](&signed_value)))
pub static func unsigned(value: u64) -> interop.Reference[Value]!:
return try own_value(try scalar_bits(DType.uint64, value))
pub static func number(value: f64) -> interop.Reference[Value]!:
var number = value
return try own_value(try scalar_bits(DType.float64, memory.read[u64](&number)))
pub static func lerp(left: interop.Reference[Value], right: interop.Reference[Value], amount: f64) -> interop.Reference[Value]!:
let a = (try left.get()).view_data()
let b = (try right.get()).view_data()
if not same_shape(a, b) or not floating(a.kind):
return try own_value(try a.copy())
var output = try value_header(a.kind, a.shape.span())
errdefer output.close()
for i in 0..<a.count():
let first = try a.number(i)
let last = try b.number(i)
let delta = last - first if a.kind == DType.float64 else (f64)((f32)last - (f32)first)
try append_number(&output.payload, a.kind, first + delta * amount)
let owned = output
output = ValueData()
return try own_value(owned)
pub func equals(other: interop.Reference[Value]) -> bool!:
return equal_values(&self.data, (try other.get()).view_data())
pub static func boolean(value: bool) -> interop.Reference[Value]!:
return try own_value(try scalar_bits(DType.boolean, 1 if value else 0))
pub static func text(value: str) -> interop.Reference[Value]!:
var data = ValueData(kind = DType.string)
data.payload.sized(value) catch failure:
data.close()
error(failure.code, failure.message)
return try own_value(data)
## Numeric payload bytes are little-endian, shape is row-major. Input is copied.
pub static func array(dtype: DType, shape: const i64[], bytes: const u8[]) -> interop.Reference[Value]!:
var scratch: u32[16]
let dimensions = try checked_shape(shape, scratch)
if dtype == DType.string:
error(invalid, "use Value.strings for string arrays")
let count = try shape_count(dimensions)
if count > max_bytes // dtype_width(dtype) or bytes.length != count * dtype_width(dtype):
error(invalid, "Prism array payload does not match its shape and dtype")
var data = try value_header(dtype, dimensions)
data.payload = copied(bytes) catch failure:
data.close()
error(failure.code, failure.message)
return try own_value(data)
pub static func strings(shape: const i64[], values: const str[]) -> interop.Reference[Value]!:
var scratch: u32[16]
let dimensions = try checked_shape(shape, scratch)
if try shape_count(dimensions) != values.length:
error(invalid, "Prism string count does not match its shape")
var data = try value_header(DType.string, dimensions)
for value in values:
data.payload.sized(value) catch failure:
data.close()
error(failure.code, failure.message)
return try own_value(data)
pub static func numbers(shape: const i64[], values: const f64[], dtype: DType = DType.float64) -> interop.Reference[Value]!:
var scratch: u32[16]
let dimensions = try checked_shape(shape, scratch)
if not floating(dtype) or try shape_count(dimensions) != values.length:
error(invalid, "expected a floating dtype and matching Prism shape")
var data = try value_header(dtype, dimensions)
for value in values:
append_number(&data.payload, dtype, value) catch failure:
data.close()
error(failure.code, failure.message)
return try own_value(data)
pub func dtype() -> DType:
return self.data.kind
pub func rank() -> usize:
return self.data.shape.count
pub func dimension(index: usize) -> usize!:
if index >= self.data.shape.count:
error(invalid, "Prism dimension index is out of bounds")
return self.data.shape.items[index]
pub func count() -> usize:
return self.data.count()
pub func bytes() -> const u8[]:
return self.data.payload.bytes()
pub func number_at(index: usize = 0) -> f64!:
return try self.data.number(index)
pub func integer_at(index: usize = 0) -> i64!:
let kind = self.data.kind
if index >= self.count() or (kind != DType.int8 and kind != DType.int16 and kind != DType.int32 and kind != DType.int64):
error(invalid, "expected a signed integer Prism component")
return self.data.signed(index)
pub func unsigned_at(index: usize = 0) -> u64!:
if index >= self.count() or floating(self.data.kind) or self.data.kind == DType.string:
error(invalid, "expected an integer Prism component")
return self.data.bits(index)
pub func text_at(index: usize = 0) -> str!:
if self.data.kind != DType.string or index >= self.count():
error(invalid, "expected a Prism string component")
var reader = Reader(data = self.data.payload.bytes())
for i in 0..<index:
_ = try reader.text()
return try reader.text()
pub func close():
self.data.close()
pub let value_type = interop.Type[Value]("Prism value", Value.close, closeable = true)
pub func own_value(data: ValueData) -> interop.Reference[Value]!:
var owned = data
let native = new Value() in memory.heap catch failure:
owned.close()
error(failure.code, failure.message)
native.data.close()
native.data = owned
return try interop.Reference[Value].adopt(native, value_type, memory.heap)
pub func checked_shape(shape: const i64[], output: u32[]) -> const u32[]!:
if shape.length > 16:
error(limit_exceeded, "Prism values support at most 16 dimensions")
for i in 0..<shape.length:
if shape[i] < 0 or shape[i] > 4294967295:
error(invalid, "Prism dimensions must fit uint32")
output[i] = (u32)shape[i]
return output[..<shape.length]
pub func same_shape(a: const ValueData*, b: const ValueData*) -> bool:
if a.kind != b.kind or a.shape.count != b.shape.count:
return false
for i in 0..<a.shape.count:
if a.shape.items[i] != b.shape.items[i]:
return false
return true
pub func equal_values(a: const ValueData*, b: const ValueData*) -> bool:
return same_shape(a, b) and a.payload.bytes() == b.payload.bytes()