Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-base

Source of release 0.41.0

0.41.0 / src / back / native / arm64 / vectors.lucb

#==============================================================================================
#
#   arm64/vectors - Vectors
#
#   DESCRIPTION:
#       Lane-wise vector operations (§5.12).
#
#==============================================================================================
extend Generator:
    # mark: Vectors (§5.12) ====================================================================

    ## A vector instruction on NEON: the operands come from their homes into v16 and v17,
    ## the result computes into v16 (v18 for a checked form) and goes to its home. The lane
    ## shape picks the arrangement, `.4s` or `.8b`; an eight-byte vector is the low half
    ## of the register, loaded and stored as `d`.
    func vector(n: ir.Instr) -> !:
        let shape = n.extra
        let bytes = ir.vector_bytes(shape)
        let arr = arrangement(shape)
        let all = ".16b" if bytes == 16 else ".8b"
        # Full-width operations can read and write their allocated registers
        # directly. Narrow and checked vectors keep the specialized sequences.
        if bytes == 16 and try self.full_vector(n, arr, all):
            return
        match n.op:
            .loadv:
                try self.address_operand("x9", n.a)
                try self.writer.line3("ldr ", vector_scratch(0, bytes), ", [x9]")
                try self.vector_result(n.dst, 0, bytes)
            .storev:
                try self.vector_source(0, n.a, bytes)
                try self.address_operand("x9", n.b)
                try self.writer.line3("str ", vector_scratch(0, bytes), ", [x9]")
            .vsplat:
                try self.splat(n, shape, arr)
                try self.vector_result(n.dst, 0, bytes)
            .vneg, .vnot:
                try self.vector_source(0, n.a, bytes)
                let form = arr if n.op == ir.Op.vneg else all
                try self.writer.line("fneg v16" if n.op == ir.Op.vneg else "mvn v16")
                try self.writer.put(form)
                try self.writer.put(", v16")
                try self.writer.put(form)
                try self.vector_result(n.dst, 0, bytes)
            .vshl, .vshr:
                try self.vector_shift(n, arr, bytes)
            .vaddo, .vsubo:
                try self.vector_checked(n, arr, all, bytes)
            _:
                try self.vector_source(0, n.a, bytes)
                try self.vector_source(1, n.b, bytes)
                let bitwise = n.op == ir.Op.vand or n.op == ir.Op.vor or n.op == ir.Op.vxor
                let form = all if bitwise else arr
                try self.writer.line(vector_mnemonic(n.op, ir.lanes_signed(shape)))
                try self.writer.put(" v16")
                try self.writer.put(form)
                try self.writer.put(", v16")
                try self.writer.put(form)
                try self.writer.put(", v17")
                try self.writer.put(form)
                try self.vector_result(n.dst, 0, bytes)

    ## A sixteen-byte load, store, or lane-wise arithmetic straight between the allocated
    ## registers; false for an operation that takes the scratch sequences instead.
    func full_vector(n: ir.Instr, arr: str, all: str) -> bool!:
        if n.op == ir.Op.loadv:
            let address = try self.source(n.a, ir.Class.l, "x9")
            let d = self.target(n.dst, ir.Class.v, "q16")
            try self.writer.line4("ldr ", d, ", [", address)
            try self.writer.put("]")
            try self.finish(n.dst, ir.Class.v, d)
            return true
        if n.op == ir.Op.storev:
            let value = try self.source(n.a, ir.Class.v, "q16")
            let address = try self.source(n.b, ir.Class.l, "x9")
            try self.writer.line4("str ", value, ", [", address)
            try self.writer.put("]")
            return true
        if n.op == ir.Op.vadd or n.op == ir.Op.vsub or n.op == ir.Op.vmul or n.op == ir.Op.vdiv or n.op == ir.Op.vand or n.op == ir.Op.vor or n.op == ir.Op.vxor or n.op == ir.Op.vaddw or n.op == ir.Op.vsubw or n.op == ir.Op.vmulw:
            let a = try self.source(n.a, ir.Class.v, "q16")
            let b = try self.source(n.b, ir.Class.v, "q17")
            let d = self.target(n.dst, ir.Class.v, "q16")
            let form = all if n.op == ir.Op.vand or n.op == ir.Op.vor or n.op == ir.Op.vxor else arr
            try self.writer.line(vector_mnemonic(n.op, ir.lanes_signed(n.extra)))
            try self.writer.put(" ")
            try self.writer.put(vector_view(d))
            try self.writer.put(form)
            try self.writer.put(", ")
            try self.writer.put(vector_view(a))
            try self.writer.put(form)
            try self.writer.put(", ")
            try self.writer.put(vector_view(b))
            try self.writer.put(form)
            try self.finish(n.dst, ir.Class.v, d)
            return true
        return false

    ## `v << count` and `v >> count`: the count, one scalar, spread over the lanes; a right
    ## shift is a left shift by the negated count, arithmetic or logical by the lanes'
    ## signedness.
    func vector_shift(n: ir.Instr, arr: str, bytes: u32) -> !:
        let shape = n.extra
        try self.vector_source(0, n.a, bytes)
        let wide = ir.lane_bits(shape) == 64
        let count = try self.source(n.b, ir.Class.l if wide else ir.Class.w, "x9" if wide else "w9")
        try self.writer.line("dup v17")
        try self.writer.put(arr)
        try self.writer.put(", ")
        try self.writer.put(count)
        if n.op == ir.Op.vshr:
            try self.writer.line("neg v17")
            try self.writer.put(arr)
            try self.writer.put(", v17")
            try self.writer.put(arr)
        try self.writer.line("sshl v16" if ir.lanes_signed(shape) else "ushl v16")
        try self.writer.put(arr)
        try self.writer.put(", v16")
        try self.writer.put(arr)
        try self.writer.put(", v17")
        try self.writer.put(arr)
        try self.vector_result(n.dst, 0, bytes)

    ## Checked `+` and `-` over lanes: the wrapping result beside the saturating one; a lane
    ## that differs overflowed, and traps.
    func vector_checked(n: ir.Instr, arr: str, all: str, bytes: u32) -> !:
        try self.vector_source(0, n.a, bytes)
        try self.vector_source(1, n.b, bytes)
        let add = n.op == ir.Op.vaddo
        try self.writer.line("add v18" if add else "sub v18")
        try self.writer.put(arr)
        try self.writer.put(", v16")
        try self.writer.put(arr)
        try self.writer.put(", v17")
        try self.writer.put(arr)
        let signed = ir.lanes_signed(n.extra)
        try self.writer.line(("sqadd v16" if signed else "uqadd v16") if add else ("sqsub v16" if signed else "uqsub v16"))
        try self.writer.put(arr)
        try self.writer.put(", v16")
        try self.writer.put(arr)
        try self.writer.put(", v17")
        try self.writer.put(arr)
        try self.writer.line("cmeq v16")
        try self.writer.put(arr)
        try self.writer.put(", v16")
        try self.writer.put(arr)
        try self.writer.put(", v18")
        try self.writer.put(arr)
        try self.writer.line("uminv b16, v16")
        try self.writer.put(all)
        try self.writer.line("fmov w9, s16")
        try self.writer.line("cbnz w9, 1f")
        try self.trap_text("integer overflow", n.extra2)
        try self.writer.put("\n1:")
        try self.vector_result(n.dst, 2, bytes)

    ## `T[N](x)`: the scalar into every lane of v16.
    func splat(n: ir.Instr, shape: u32, arr: str) -> !:
        if ir.lanes_float(shape):
            let cls = ir.Class.d if ir.lane_bits(shape) == 64 else ir.Class.s
            try self.operand(scratch_register(cls, 0), cls, n.a)
            try self.writer.line("dup v16")
            try self.writer.put(arr)
            try self.writer.put(", v16.d[0]" if cls == ir.Class.d else ", v16.s[0]")
            return
        let wide = ir.lane_bits(shape) == 64
        let scalar = try self.source(n.a, ir.Class.l if wide else ir.Class.w, "x9" if wide else "w9")
        try self.writer.line("dup v16")
        try self.writer.put(arr)
        try self.writer.put(", ")
        try self.writer.put(scalar)

    ## A vector temporary into scratch register `k`: from the register it lives in, else
    ## from its home.
    func vector_source(k: u32, r: ir.Ref, bytes: u32) -> !:
        try self.load_temp(vector_scratch(k, bytes), ir.Class.v, r.index)

    ## Scratch register `k` into vector temporary `dst`, where it lives.
    func vector_result(dst: u32, k: u32, bytes: u32) -> !:
        if dst == 0:
            return
        try self.store_temp(vector_scratch(k, bytes), dst)