Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-base

Source of release 0.46.0

0.46.0 / src / back / native / x86_64 / vectors.lucb

#==============================================================================================
#
#   x86_64/vectors - Vectors
#
#   DESCRIPTION:
#       Lane-wise vector operations (§5.12).
#
#==============================================================================================
extend Generator:
    # mark: Vectors (§5.12) ====================================================================

    ## A vector instruction on SSE2: the operands come from their homes into the first two
    ## scratch registers (`reg`), the result computes into the first (the third for a
    ## checked form) and goes to its home. An eight-byte vector is the low half of the
    ## register, moved as a quadword; the upper lanes then hold zeros, which no instruction
    ## here reads.
    func vector(n: ir.Instr) -> !:
        let shape = n.extra
        let bytes = ir.vector_bytes(shape)
        let bits = ir.lane_bits(shape)
        let signed = ir.lanes_signed(shape)
        let first = self.reg(ir.Class.v, 0)
        let second = self.reg(ir.Class.v, 1)
        if try self.float_lanes(n, first):
            return
        if bytes == 16:
            if n.op == ir.Op.loadv:
                try self.address_operand("%r10", n.a)
                let d = self.target(n.dst, ir.Class.v, first)
                # float lanes move in the float domain: no bypass delay before an FP operation
                try self.writer.line3("movups (%r10), " if ir.lanes_float(shape) else "movdqu (%r10), ", d, "")
                try self.finish(n.dst, ir.Class.v, d)
                return
            if n.op == ir.Op.storev:
                let value = try self.source(n.a, ir.Class.v, first)
                try self.address_operand("%r10", n.b)
                try self.writer.line3("movups " if ir.lanes_float(shape) else "movdqu ", value, ", (%r10)")
                return
            if n.op == ir.Op.vadd or n.op == ir.Op.vsub or n.op == ir.Op.vmul or n.op == ir.Op.vdiv or n.op == ir.Op.vand or n.op == ir.Op.vor or n.op == ir.Op.vxor or n.op == ir.Op.vaddw or n.op == ir.Op.vsubw or n.op == ir.Op.vmulw:
                let a = try self.source(n.a, ir.Class.v, first)
                let b = try self.source(n.b, ir.Class.v, second)
                var d = self.target(n.dst, ir.Class.v, first)
                # SSE uses a destructive destination. Preserve a distinct right
                # operand when the allocator gave its register to the result.
                if d == b and d != a:
                    d = first
                if a != d:
                    try self.writer.line4("movaps " if ir.lanes_float(shape) else "movdqa ", a, ", ", d)
                let mnemonic = "pmulld" if n.op == ir.Op.vmulw and bits == 32 else vector_mnemonic(n.op, shape)
                try self.writer.line4(mnemonic, " ", b, ", ")
                try self.writer.put(d)
                try self.finish(n.dst, ir.Class.v, d)
                return
        match n.op:
            .loadv:
                try self.address_operand("%r10", n.a)
                try self.writer.line3(vector_move(bytes), "(%r10), ", first)
                try self.vector_result(n.dst, first, bytes)
            .storev:
                try self.vector_source(n.a, first, bytes)
                try self.address_operand("%r10", n.b)
                try self.writer.line4(vector_move(bytes), first, ", ", "(%r10)")
            .vsplat:
                try self.splat(n, shape, first)
                try self.vector_result(n.dst, first, bytes)
            .vneg:
                # the sign bit of every lane flipped
                try self.vector_source(n.a, first, bytes)
                try self.writer.line4("xorps ", self.shared(".Lvsign64" if bits == 64 else ".Lvsign32"), "(%rip), ", first)
                try self.vector_result(n.dst, first, bytes)
            .vnot:
                try self.vector_source(n.a, first, bytes)
                try self.writer.line4("pcmpeqd ", second, ", ", second)
                try self.writer.line4("pxor ", second, ", ", first)
                try self.vector_result(n.dst, first, bytes)
            .vshl, .vshr:
                # the count, one scalar, in the second scratch register's low quadword
                try self.vector_source(n.a, first, bytes)
                try self.operand("%rax", ir.Class.l, n.b)
                try self.writer.line3("movq %rax, ", second, "")
                var mnemonic = "psllw"
                if n.op == ir.Op.vshl:
                    mnemonic = "psllw" if bits == 16 else ("pslld" if bits == 32 else "psllq")
                elif signed:
                    mnemonic = "psraw" if bits == 16 else "psrad"
                else:
                    mnemonic = "psrlw" if bits == 16 else ("psrld" if bits == 32 else "psrlq")
                try self.writer.line5(mnemonic, " ", second, ", ", first)
                try self.vector_result(n.dst, first, bytes)
            .vaddo, .vsubo:
                try self.checked_lanes(n, shape)
            _:
                try self.vector_source(n.a, first, bytes)
                try self.vector_source(n.b, second, bytes)
                if n.op == ir.Op.vmulw and bits == 32:
                    # x86-64-v2: the doubleword multiply of SSE4.1
                    try self.writer.line4("pmulld ", second, ", ", first)
                else:
                    try self.writer.line5(vector_mnemonic(n.op, shape), " ", second, ", ", first)
                try self.vector_result(n.dst, first, bytes)

    ## Float lanes straight between their registers: two singles to and from memory as the
    ## register's low quadword, and a float broadcast into every lane; false for anything
    ## else.
    func float_lanes(n: ir.Instr, first: str) -> bool!:
        let shape = n.extra
        let bytes = ir.vector_bytes(shape)
        let bits = ir.lane_bits(shape)
        if not ir.lanes_float(shape):
            return false
        if bytes == 8 and (n.op == ir.Op.loadv or n.op == ir.Op.storev):
            # two singles, the register's low quadword, between their registers and memory
            try self.address_operand("%r10", n.b if n.op == ir.Op.storev else n.a)
            if n.op == ir.Op.loadv:
                let d = self.target(n.dst, ir.Class.v, first)
                try self.writer.line3("movsd (%r10), ", d, "")
                try self.finish(n.dst, ir.Class.v, d)
            else:
                let value = try self.source(n.a, ir.Class.v, first)
                try self.writer.line3("movsd ", value, ", (%r10)")
            return true
        if bytes == 16 and n.op == ir.Op.vsplat:
            # a float straight from where it lives into every lane of the result's register
            let cls = ir.Class.d if bits == 64 else ir.Class.s
            let v = try self.source(n.a, cls, self.reg(cls, 1))
            let d = self.target(n.dst, ir.Class.v, first)
            if v != d:
                try self.writer.line4("movaps ", v, ", ", d)
            try self.writer.line4("unpcklpd " if bits == 64 else "shufps $0, ", d, ", ", d)
            try self.finish(n.dst, ir.Class.v, d)
            return true
        return false

    ## The lanes the optimiser packs from scalars (`opt/slp`), between the allocated
    ## registers: a scalar into lane 0 (`movsd`/`movss`, merging) or lane 1 (`unpcklpd`, or
    ## `unpcklps` for a two-lane single vector, the only shapes the optimiser inserts
    ## into), a lane out to a scalar (lane 0 is the register itself; `unpckhpd` or `shufps`
    ## bring another down), and the square root of every lane. An inserted scalar sharing
    ## the result's register is moved aside before the vector it joins is copied in.
    func packed_lanes(n: ir.Instr) -> !:
        let shape = n.extra
        let wide = ir.lane_bits(shape) == 64
        let cls = ir.Class.d if wide else ir.Class.s
        if n.op == ir.Op.vsqrt or n.op == ir.Op.vexts or n.op == ir.Op.vtruncd:
            let a = try self.source(n.a, ir.Class.v, self.reg(ir.Class.v, 0))
            let d = self.target(n.dst, ir.Class.v, self.reg(ir.Class.v, 0))
            var mnemonic = "sqrtpd " if wide else "sqrtps "
            if n.op == ir.Op.vexts:
                mnemonic = "cvtps2pd "
            elif n.op == ir.Op.vtruncd:
                mnemonic = "cvtpd2ps "
            try self.writer.line4(mnemonic, a, ", ", d)
            try self.finish(n.dst, ir.Class.v, d)
            return
        if n.op == ir.Op.vextract:
            let a = try self.source(n.a, ir.Class.v, self.reg(ir.Class.v, 0))
            let d = self.target(n.dst, cls, self.reg(cls, 0))
            if a != d:
                try self.writer.line4("movaps ", a, ", ", d)
            if n.extra2 > 0 and wide:
                try self.writer.line4("unpckhpd ", d, ", ", d)
            elif n.extra2 > 0:
                try self.writer.line("shufps $")
                try self.writer.put_u64((u64)n.extra2)
                try self.put2(", ", d)
                try self.put2(", ", d)
            try self.finish(n.dst, cls, d)
            return
        var b = try self.source(n.b, cls, self.reg(cls, 1))
        let d = self.target(n.dst, ir.Class.v, self.reg(ir.Class.v, 0))
        if n.a.kind == ir.RefKind.none_ and n.extra2 == 0:
            # the other lanes are anything: the scalar's own move
            if b != d:
                try self.writer.line4("movaps ", b, ", ", d)
            try self.finish(n.dst, ir.Class.v, d)
            return
        if b == d:
            try self.writer.line4("movaps ", b, ", ", self.reg(cls, 1))
            b = self.reg(cls, 1)
        if n.a.kind != ir.RefKind.none_:
            let a = try self.source(n.a, ir.Class.v, self.reg(ir.Class.v, 0))
            if a != d:
                try self.writer.line4("movaps ", a, ", ", d)
        if n.extra2 == 0:
            try self.writer.line4("movsd " if wide else "movss ", b, ", ", d)
        else:
            try self.writer.line4("unpcklpd " if wide else "unpcklps ", b, ", ", d)
        try self.finish(n.dst, ir.Class.v, d)

    ## `vaddo`/`vsubo`: the wrapping result in the third scratch register; a lane that
    ## overflowed traps. With `a` and `b` in the first two: bytes and halfwords compare the
    ## wrapping result with the saturating one; doublewords read the sign of the overflow
    ## test, `(a ^ r) & (b ^ r)` for an add, `(a ^ b) & (a ^ r)` for a subtract, or the
    ## unsigned order of the operands through a sign flip. `a` and `b` are spent on it.
    func checked_lanes(n: ir.Instr, shape: u32) -> !:
        let bytes = ir.vector_bytes(shape)
        let bits = ir.lane_bits(shape)
        let signed = ir.lanes_signed(shape)
        let add = n.op == ir.Op.vaddo
        let a = self.reg(ir.Class.v, 0)
        let b = self.reg(ir.Class.v, 1)
        let r = self.reg(ir.Class.v, 2)
        try self.vector_source(n.a, a, bytes)
        try self.vector_source(n.b, b, bytes)
        try self.writer.line4("movdqa ", a, ", ", r)
        try self.writer.line5(lane_mnemonic("padd" if add else "psub", bits), " ", b, ", ", r)
        if bits <= 16:
            let sat = ("padds" if signed else "paddus") if add else ("psubs" if signed else "psubus")
            try self.writer.line5(lane_mnemonic(sat, bits), " ", b, ", ", a)
            try self.writer.line4("pcmpeqb ", r, ", ", a)
            try self.writer.line3("pmovmskb ", a, ", %eax")
            if bytes == 16:
                try self.writer.line("cmpl $65535, %eax")
            else:
                try self.writer.line("andl $255, %eax")
                try self.writer.line("cmpl $255, %eax")
            try self.writer.line("je 1f")
        elif signed:
            # the overflow test's sign bits, into `a`
            if add:
                try self.writer.line4("pxor ", r, ", ", a)
                try self.writer.line4("pxor ", r, ", ", b)
            else:
                try self.writer.line4("pxor ", a, ", ", b)
                try self.writer.line4("pxor ", r, ", ", a)
            try self.writer.line4("pand ", b, ", ", a)
            try self.writer.line3("pmovmskb ", a, ", %eax")
            try self.writer.line("andl $34952, %eax" if bytes == 16 else "andl $136, %eax")
            try self.writer.line("je 1f")
        else:
            # unsigned: an add overflowed where r < a, a subtract where a < b; the sign
            # flip makes the signed compare an unsigned one
            let sign = self.shared(".Lvsign32")
            try self.writer.line4("pxor ", sign, "(%rip), ", a)
            if add:
                # b = r with its signs flipped; a > b where r < a
                try self.writer.line4("movdqa ", r, ", ", b)
                try self.writer.line4("pxor ", sign, "(%rip), ", b)
                try self.writer.line4("pcmpgtd ", b, ", ", a)
            else:
                # b > a where a < b
                try self.writer.line4("pxor ", sign, "(%rip), ", b)
                try self.writer.line4("pcmpgtd ", a, ", ", b)
                try self.writer.line4("movdqa ", b, ", ", a)
            try self.writer.line3("pmovmskb ", a, ", %eax")
            try self.writer.line("andl $65535, %eax" if bytes == 16 else "andl $255, %eax")
            try self.writer.line("je 1f")
        try self.trap_text("integer overflow", n.extra2)
        try self.writer.put("\n1:")
        try self.vector_result(n.dst, r, bytes)

    ## `T[N](x)`: the scalar into every lane of `into`.
    func splat(n: ir.Instr, shape: u32, into: str) -> !:
        let bits = ir.lane_bits(shape)
        if ir.lanes_float(shape):
            if bits == 64:
                try self.operand(into, ir.Class.d, n.a)
                try self.writer.line4("unpcklpd ", into, ", ", into)
            else:
                try self.operand(into, ir.Class.s, n.a)
                try self.writer.line4("shufps $0, ", into, ", ", into)
            return
        if bits == 64:
            try self.operand("%rax", ir.Class.l, n.a)
            try self.writer.line3("movq %rax, ", into, "")
            try self.writer.line4("punpcklqdq ", into, ", ", into)
            return
        try self.operand("%eax", ir.Class.w, n.a)
        try self.writer.line3("movd %eax, ", into, "")
        if bits == 8:
            try self.writer.line4("punpcklbw ", into, ", ", into)
        if bits <= 16:
            try self.writer.line4("punpcklwd ", into, ", ", into)
        try self.writer.line4("pshufd $0, ", into, ", ", into)

    ## A vector temporary into `reg`: from the register it lives in, else from its home.
    func vector_source(r: ir.Ref, reg: str, bytes: u32) -> !:
        let code = self.fr.temp_regs.get((usize)r.index - 1)
        if code != 0:
            try self.writer.line4("movdqa ", self.class_name(code, ir.Class.v), ", ", reg)
            return
        try self.writer.line(vector_move(bytes))
        try self.frame_operand(self.temp_home(r.index))
        try self.writer.put(", ")
        try self.writer.put(reg)

    ## `reg` into vector temporary `dst`, where it lives.
    func vector_result(dst: u32, reg: str, bytes: u32) -> !:
        if dst == 0:
            return
        let code = self.fr.temp_regs.get((usize)dst - 1)
        if code != 0:
            try self.writer.line4("movdqa ", reg, ", ", self.class_name(code, ir.Class.v))
            return
        try self.writer.line3(vector_move(bytes), reg, ", ")
        try self.frame_operand(self.temp_home(dst))