dymokomi/luce-geocore
Source of release 0.7.8
0.7.8 / src / core / attribute_ops.lucb
## Whole-column attribute operations for topology changes: remapping through
## parent maps and joining two meshes' attributes. Each writes one new column
## per attribute with typed loops, never per-element f64 conversions.
import memory
from luce_std import parallel
import errors
from core.attributes import Attribute, DataType, Domain, Storage, is_integer, scalar_bytes
from core.strings import TableBuilder, table_count, table_text
from core.shared import Column
## `source` over a new domain of `elements`, element i taking its value from
## element `parents[i]` (-1: zero, or "" for text). An empty map keeps the
## indexing. Single and detail attributes do not depend on elements and are
## shared. Large columns are gathered in parallel.
pub func remapped(source: const Attribute*, parents: const i32[], elements: usize) -> Attribute!:
if source.storage == Storage.single or source.domain == Domain.detail:
var shared = source.share()
shared.elements = elements
return shared
if parents.length == 0 and elements == source.elements:
return source.share()
if parents.length != elements: error(errors.invalid, "an attribute parent map needs one parent per element")
if source.storage == Storage.ragged:
return try remapped_ragged(source, parents, elements)
var result = try Attribute.create(source.name(), source.domain, source.kind, source.width, elements)
errdefer result.release()
result.flags = source.flags
result.role = source.role
result.table = source.table.share()
let input = source.data.view()
let output = result.data.span()
if source.kind == DataType.boolean:
var bits = BitWork(input, output, parents)
try parallel.for_each_range((elements + 63) // 64, 4096, (void*)&bits, gather_bits)
elif source.kind == DataType.int8:
let values = typed_mut[i8](output)
let old = typed[i8](input)
for at in 0..<elements:
let parent = parents[at]
for component in 0..<source.width:
values[at * source.width + component] = old[(usize)parent * source.width + component] if parent >= 0 else 0
else:
let words = source.tuple_bytes() // 4
var work = GatherWork(typed[u32](input), typed_mut[u32](output), words, parents, 0xFFFFFFFF if source.kind == DataType.text else 0)
try parallel.for_each_range(elements, 65536, (void*)&work, gather_words)
return result
## An array attribute through a parent map: each element copies its parent's items
## (an element without a parent gets an empty array).
func remapped_ragged(source: const Attribute*, parents: const i32[], elements: usize) -> Attribute!:
var offsets = try Column[u8].allocate((elements + 1) * 4)
errdefer offsets.release()
let bounds = u32[]((u32*)offsets.span().data, elements + 1)
var total: u32 = 0
for at in 0..<elements:
bounds[at] = total
let parent = parents[at]
if parent >= 0: total += (u32)source.item_count((usize)parent)
bounds[elements] = total
let item = source.width * 4
var values = try Column[u8].allocate((usize)total * item)
errdefer values.release()
var work = RaggedWork(source, parents, bounds, values.span(), item)
try parallel.for_each_range(elements, 16384, (void*)&work, gather_ragged)
return try Attribute.ragged(source.name(), source.domain, source.kind, source.width, offsets, values)
## An array gather's inputs: each element's items go to its new offset.
struct RaggedWork:
let source: const Attribute*
let parents: const i32[]
let bounds: const u32[]
let output: u8[]
## Bytes per item.
let item: usize
func gather_ragged(context: void*, first: usize, end: usize, worker: usize):
_ = worker
let work = (RaggedWork*)context
let input = work.source.data.view()
let item = work.item
for at in first..<end:
let parent = work.parents[at]
if parent < 0: continue
let count = work.source.item_count((usize)parent)
let from_item = work.source.first_item((usize)parent)
let to = (usize)work.bounds[at] * item
memory.copy(work.output[to..<to + count * item], input[from_item * item..<(from_item + count) * item], count * item)
## The array attribute of a concatenation: each piece's elements' arrays in turn, empty
## arrays where a piece lacks it. Pieces must agree on item kind and width.
func concatenated_ragged(first: const Attribute*, pieces: const Piece[]) -> Attribute!:
var elements: usize = 0
var items: usize = 0
for piece in pieces:
elements += piece.count
if let attribute = piece.attribute:
if attribute.storage != Storage.ragged or attribute.kind != first.kind or attribute.width != first.width:
error(errors.invalid, "Merge inputs have conflicting attribute schemas")
items += (usize)memory.read[u32]((void*)&attribute.offsets.view()[attribute.elements * 4])
var offsets = try Column[u8].allocate((elements + 1) * 4)
errdefer offsets.release()
let bounds = u32[]((u32*)offsets.span().data, elements + 1)
let item = first.width * 4
var values = try Column[u8].allocate(items * item)
errdefer values.release()
var element: usize = 0
var written: usize = 0
for piece in pieces:
for local in 0..<piece.count:
bounds[element] = (u32)written
if let attribute = piece.attribute:
let count = attribute.item_count(local)
let from_item = attribute.first_item(local)
memory.copy(values.span()[written * item..<(written + count) * item], attribute.data.view()[from_item * item..<(from_item + count) * item], count * item)
written += count
element += 1
bounds[elements] = (u32)written
return try Attribute.ragged(first.name(), first.domain, first.kind, first.width, offsets, values)
struct GatherWork:
let input: const u32[]
let output: u32[]
## 32-bit words per tuple.
let words: usize
let parents: const i32[]
## The value of each word of an element without a parent.
let fill: u32
func gather_words(context: void*, first: usize, end: usize, worker: usize):
let work = (GatherWork*)context
let words = work.words
for at in first..<end:
let parent = work.parents[at]
let target = at * words
if parent < 0:
for word in 0..<words: work.output[target + word] = work.fill
else:
let origin = (usize)parent * words
for word in 0..<words: work.output[target + word] = work.input[origin + word]
struct BitWork:
let input: const u8[]
let output: u8[]
let parents: const i32[]
## Output words first..<end: each takes the bits of its elements' parents.
func gather_bits(context: void*, first: usize, end: usize, worker: usize):
let work = (BitWork*)context
let output = typed_mut[u64](work.output)
let input = typed[u64](work.input)
for word in first..<end:
var value: u64 = 0
let stop = word * 64 + 64 if work.parents.length > word * 64 + 64 else work.parents.length
for at in word * 64..<stop:
let parent = work.parents[at]
if parent >= 0 and (input[(usize)parent >> 6] >> (u64)((usize)parent & 63)) & 1 != 0:
value = value | ((u64)1 << (u64)(at & 63))
output[word] = value
## One input of a concatenation: an attribute (or none where that input
## lacks it) and the elements that input contributes.
pub struct Piece:
pub let attribute: const Attribute*?
pub let count: usize
## The attribute of a concatenation: each piece's elements in turn, zero (or
## "") where a piece lacks the attribute. The pieces must agree on width,
## integer-ness and text; the widest scalar type wins. A detail attribute
## keeps the first value; single storage expands when the pieces differ.
pub func concatenated(pieces: const Piece[]) -> Attribute!:
var model: const Attribute*? = none
var kind = DataType.boolean
for piece in pieces:
let attribute = piece.attribute else continue
if let first = model:
if attribute.width != first.width or is_integer(attribute.kind) != is_integer(first.kind) or (attribute.kind == DataType.text) != (first.kind == DataType.text) or (attribute.kind == DataType.boolean) != (first.kind == DataType.boolean):
error(errors.invalid, "Merge inputs have conflicting attribute schemas")
if scalar_bytes(attribute.kind) > scalar_bytes(kind): kind = attribute.kind
else:
model = attribute
kind = attribute.kind
let first = model else error(errors.invalid, "a concatenation needs the attribute on some input")
if first.storage == Storage.ragged:
return try concatenated_ragged(first, pieces)
if first.domain == Domain.detail:
return first.share()
var total: usize = 0
for piece in pieces: total += piece.count
var result = try Attribute.create(first.name(), first.domain, kind, first.width, total)
errdefer result.release()
result.flags = first.flags
result.role = first.role
let output = result.data.span()
memory.set(output, 0)
if kind == DataType.text:
try concatenate_text(&result, pieces)
return result
var at: usize = 0
for piece in pieces:
let base = at
at += piece.count
let attribute = piece.attribute else continue
if attribute.kind == kind and attribute.storage == Storage.array and kind != DataType.boolean:
let size = attribute.tuple_bytes()
memory.copy(output[base * size..<(base + piece.count) * size], attribute.data.view(), piece.count * size)
continue
for element in 0..<piece.count:
for component in 0..<first.width:
write_number(output, kind, (base + element) * first.width + component, attribute.number(element, component))
return result
## The text of a concatenation: one table of every piece's strings, each
## piece's indices mapped into it.
func concatenate_text(result: Attribute*, pieces: const Piece[]) -> !:
var table = TableBuilder()
defer table.close()
let output = typed_mut[i32](result.data.span())
var at: usize = 0
for piece in pieces:
let base = at
at += piece.count
if piece.attribute == none:
for element in base..<base + piece.count: output[element] = -1
continue
let attribute = piece.attribute else continue
let strings = attribute.table.view()
let mapping = try new i32[table_count(strings)] --- in memory.heap
defer free(mapping) in memory.heap
for index in 0..<mapping.length: mapping[index] = try table.intern(table_text(strings, index))
let values = attribute.data.view()
for element in 0..<piece.count:
let index = memory.read[i32]((void*)&values[(0 if attribute.storage == Storage.single else element) * 4])
output[base + element] = mapping[(usize)index] if index >= 0 and (usize)index < mapping.length else -1
result.table = try table.finish()
## Write `value` as element `index` of a column of `kind`.
pub func write_number(bytes: u8[], kind: DataType, index: usize, value: f64):
match kind:
.boolean:
if value != 0.0: bytes[index // 8] = bytes[index // 8] | ((u8)1 << (u8)(index % 8))
.int8: bytes[index] = (u8)(i8)value
.int32, .text: memory.write[i32]((void*)&bytes[index * 4], (i32)value)
.int64: memory.write[i64]((void*)&bytes[index * 8], (i64)value)
.float32: memory.write[f32]((void*)&bytes[index * 4], (f32)value)
_: memory.write[f64]((void*)&bytes[index * 8], value)
## Column bytes as scalars; columns are 64-byte aligned.
pub func typed[T](bytes: const u8[]) -> const T[]:
return T[]((T*)bytes.data, bytes.length // memory.size_of(T))
pub func typed_mut[T](bytes: u8[]) -> T[]:
return T[]((T*)bytes.data, bytes.length // memory.size_of(T))