Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-kernel

Source of release 0.1.2

0.1.2 / src / engine / lanes_data.lucb

# Data movement: copies, masked assignment, select, element numbers, and indexed register
# reads and writes. Moves work by size class, so one loop serves every kind of a width.
# Lane loops assert their lengths, which lets the --opt 3 vectorizer take them.

## d = a on every live lane.
func copy_lanes(w: Worker*, ins: Instruction):
    match ins.kind.size():
        1: copy8(w.masks(ins.dst), w.masks(ins.a))
        4: copy32(w.lanes32(ins.dst), w.lanes32(ins.a))
        _: copy64(w.lanes64(ins.dst), w.lanes64(ins.a))

func copy8(d: u8[], a: const u8[]):
    assert(a.length == d.length)
    for i in 0..<d.length: d[i] = a[i]

func copy32(d: u32[], a: const u32[]):
    assert(a.length == d.length)
    for i in 0..<d.length: d[i] = a[i]

func copy64(d: u64[], a: const u64[]):
    assert(a.length == d.length)
    for i in 0..<d.length: d[i] = a[i]

## d = a on the active lanes; inactive lanes keep d.
func assign_lanes(w: Worker*, ins: Instruction):
    let on = w.on()
    if on.all:
        copy_lanes(w, ins)
        return
    match ins.kind.size():
        1: assign8(w.masks(ins.dst), w.masks(ins.a), on.exec)
        4: assign32(w.lanes32(ins.dst), w.lanes32(ins.a), on.exec)
        _: assign64(w.lanes64(ins.dst), w.lanes64(ins.a), on.exec)

func assign8(d: u8[], a: const u8[], exec: const u8[]):
    assert(a.length == d.length and exec.length == d.length)
    for i in 0..<d.length: d[i] = a[i] if exec[i] != 0 else d[i]

func assign32(d: u32[], a: const u32[], exec: const u8[]):
    assert(a.length == d.length and exec.length == d.length)
    for i in 0..<d.length: d[i] = a[i] if exec[i] != 0 else d[i]

func assign64(d: u64[], a: const u64[], exec: const u8[]):
    assert(a.length == d.length and exec.length == d.length)
    for i in 0..<d.length: d[i] = a[i] if exec[i] != 0 else d[i]

## d = b where mask a is set, else c.
func select_lanes(w: Worker*, ins: Instruction):
    let mask = w.masks(ins.a)
    match ins.kind.size():
        1: select8(w.masks(ins.dst), mask, w.masks(ins.b), w.masks(ins.c))
        4: select32(w.lanes32(ins.dst), mask, w.lanes32(ins.b), w.lanes32(ins.c))
        _: select64(w.lanes64(ins.dst), mask, w.lanes64(ins.b), w.lanes64(ins.c))

func select8(d: u8[], mask: const u8[], b: const u8[], c: const u8[]):
    assert(mask.length == d.length and b.length == d.length and c.length == d.length)
    for i in 0..<d.length: d[i] = b[i] if mask[i] != 0 else c[i]

func select32(d: u32[], mask: const u8[], b: const u32[], c: const u32[]):
    assert(mask.length == d.length and b.length == d.length and c.length == d.length)
    for i in 0..<d.length: d[i] = b[i] if mask[i] != 0 else c[i]

func select64(d: u64[], mask: const u8[], b: const u64[], c: const u64[]):
    assert(mask.length == d.length and b.length == d.length and c.length == d.length)
    for i in 0..<d.length: d[i] = b[i] if mask[i] != 0 else c[i]

## d = the lane's element number (`run` checks that every number fits the kind).
func index_lanes(w: Worker*, ins: Instruction):
    match ins.kind:
        .i32:
            let d = w.i32s(ins.dst)
            for i in 0..<d.length: d[i] = (i32)(w.first + i)
        .i64:
            let d = w.i64s(ins.dst)
            for i in 0..<d.length: d[i] = (i64)(w.first + i)
        .u32:
            let d = w.u32s(ins.dst)
            for i in 0..<d.length: d[i] = (u32)(w.first + i)
        _:
            let d = w.u64s(ins.dst)
            for i in 0..<d.length: d[i] = (u64)(w.first + i)

## The value of integer register `register` at `lane` as an index among `count`, or none
## when it is negative or not below `count`.
func index_at(w: Worker*, program: const Program*, register: u16, lane: usize, count: u16) -> usize?:
    let kind = program.kinds.at((usize)register)
    var value: u64 = 0
    match kind:
        .i32:
            let signed = w.i32s(register)[lane]
            if signed < 0: return none
            value = (u64)signed
        .i64:
            let signed = w.i64s(register)[lane]
            if signed < 0: return none
            value = (u64)signed
        .u32: value = (u64)w.u32s(register)[lane]
        _: value = w.u64s(register)[lane]
    if value >= (u64)count:
        return none
    return (usize)value

## d = register (a + b[lane]) of the `c` registers from `a`; an index out of range faults
## on an active lane and reads register `a` on an inactive one.
func pick_lanes(w: Worker*, ins: Instruction, program: const Program*) -> usize:
    let on = w.on()
    let size = ins.kind.size()
    for lane in 0..<w.n:
        var from_register = ins.a
        if let at = index_at(w, program, ins.b, lane, ins.c):
            from_register = ins.a + (u16)at
        elif on.at(lane):
            return lane
        match size:
            1: w.masks(ins.dst)[lane] = w.masks(from_register)[lane]
            4: w.lanes32(ins.dst)[lane] = w.lanes32(from_register)[lane]
            _: w.lanes64(ins.dst)[lane] = w.lanes64(from_register)[lane]
    return w.n

## Register (d + a[lane]) of the `c` registers from `d` = b, on active lanes; an index out
## of range there faults.
func place_lanes(w: Worker*, ins: Instruction, program: const Program*) -> usize:
    let on = w.on()
    let size = ins.kind.size()
    for lane in 0..<w.n:
        if not on.at(lane):
            continue
        let at = index_at(w, program, ins.a, lane, ins.c) else return lane
        let to_register = ins.dst + (u16)at
        match size:
            1: w.masks(to_register)[lane] = w.masks(ins.b)[lane]
            4: w.lanes32(to_register)[lane] = w.lanes32(ins.b)[lane]
            _: w.lanes64(to_register)[lane] = w.lanes64(ins.b)[lane]
    return w.n