Luce packages
LuceLuce BaseLuciaOS

dymokomi/luce-base

Source of release 0.41.0

0.41.0 / src / back / native / x86_64 / instructions.lucb

#==============================================================================================
#
#   x86_64/instructions - Instructions
#
#   DESCRIPTION:
#       One IR instruction at a time: moves, arithmetic, compares, loads and stores, and
#       the checked operations that trap.
#
#==============================================================================================
extend Generator:
    # mark: Instructions =======================================================================

    func instruction(n: ir.Instr, next_label: u32 = 0) -> !:
        if n.dst != 0 and self.fr.is_rematerialised(n.dst):
            # a frame address is formed where it is used
            return
        match n.op:
            .dbg:
                _ = 0
            .label:
                if (usize)n.extra < self.loop_heads.length and self.loop_heads[(usize)n.extra]:
                    # a loop starts on a sixteen-byte boundary
                    try self.writer.put("\n    .p2align 4")
                try self.writer.put("\n")
                try self.label_name(n.extra)
                try self.writer.put(":")
            .jmp:
                if n.extra == next_label:
                    return
                if (usize)n.extra < self.head_tests.length and self.head_tests[(usize)n.extra] != 0:
                    # the head's own test, falling through to what follows the jump
                    try self.fused_branch((usize)self.head_tests[(usize)n.extra] - 1, next_label)
                    return
                try self.writer.line("jmp ")
                try self.label_name(n.extra)
            .jnz:
                let cls = ir.Class.l if n.cls == ir.Class.l else ir.Class.w
                let c = try self.source(n.a, cls, self.reg(cls, 0))
                try self.writer.line5("testq " if cls == ir.Class.l else "testl ", c, ", ", c, "")
                # into whichever target comes next, one jump to the other
                if n.extra == next_label:
                    try self.writer.line("je ")
                    try self.label_name(n.extra2)
                    return
                try self.writer.line("jne ")
                try self.label_name(n.extra)
                if n.extra2 != next_label:
                    try self.writer.line("jmp ")
                    try self.label_name(n.extra2)
            .switch_:
                try self.jump_table(n)
            .ret:
                if n.extra2 > 0:
                    try self.return_words(n)
                try self.return_value(n)
            .copy:
                if n.cls == ir.Class.v:
                    let value = try self.source(n.a, ir.Class.v, self.reg(ir.Class.v, 0))
                    let d = self.target(n.dst, ir.Class.v, self.reg(ir.Class.v, 0))
                    if value != d:
                        try self.writer.line4("movdqa ", value, ", ", d)
                    try self.finish(n.dst, ir.Class.v, d)
                    return
                let d = self.target(n.dst, n.cls, self.reg(n.cls, 0))
                if n.a.kind == ir.RefKind.temp:
                    if let bits = self.fr.constant_of(n.a.index):
                        # a float constant formed where it is used: straight into the copy
                        try self.constant(d, n.cls, bits)
                        try self.finish(n.dst, n.cls, d)
                        return
                let v = try self.source(n.a, n.cls, self.reg(n.cls, 0))
                if v != d:
                    try self.writer.line4(register_move(n.cls), v, ", ", d)
                try self.finish(n.dst, n.cls, d)
            .add, .sub, .mul, .sdiv, .udiv, .srem, .urem, .and_, .or_, .xor, .shl, .lshr, .ashr:
                if ir.is_float_class(n.cls):
                    try self.float_arithmetic(n)
                else:
                    try self.arithmetic(n)
            .neg:
                try self.negate(n)
            .fsqrt, .ffloor, .fceil, .ftrunc, .fabs:
                try self.float_function(n)
            .ceq, .cne, .cslt, .csle, .csgt, .csge, .cult, .cule, .cugt, .cuge:
                if ir.is_float_class(n.cls):
                    try self.float_comparison(n)
                else:
                    try self.comparison(n)
            .loadsb, .loadub, .loadsh, .loaduh, .loadsw, .loaduw, .loadl, .loads, .loadd, .loadhf:
                let lc = ir.load_class(n.op)
                let d = self.target(n.dst, lc, self.reg(lc, 0))
                let from_slot = self.fr.promoted_slot_of(n.a)
                if from_slot != 0:
                    try self.load_promoted(n.op, from_slot, d)
                else:
                    let place = try self.memory_place(n.a, n.op)
                    if n.op == ir.Op.loadhf:
                        try self.writer.line("movzwl (%r10), %eax")
                        try self.writer.line3("movd %eax, ", d, "")
                    else:
                        try self.writer.line(load_mnemonic(n.op))
                        try self.put_place(place)
                        try self.put2(", ", d)
                try self.finish(n.dst, lc, d)
            .storeb, .storeh, .storew, .storel, .stores, .stored, .storehf:
                let cls = ir.store_class(n.op)
                let v = try self.source(n.a, cls, self.reg(cls, 0))
                let to_slot = self.fr.promoted_slot_of(n.b)
                if to_slot != 0:
                    try self.store_promoted(n.op, to_slot, v)
                else:
                    let place = try self.memory_place(n.b, n.op)
                    if n.op == ir.Op.storehf:
                        try self.writer.line3("movd ", v, ", %eax")
                        try self.writer.line("movw %ax, (%r10)")
                    else:
                        if n.op == ir.Op.storeb:
                            try self.writer.line3("movb ", byte_name(v), ", ")
                        elif n.op == ir.Op.storeh:
                            try self.writer.line3("movw ", word_name(v), ", ")
                        else:
                            try self.writer.line3(store_mnemonic(n.op), v, ", ")
                        try self.put_place(place)
            .alloca:
                # frame storage (`memory.frame`): rsp down by a multiple of sixteen, at most
                # 4096, and the new bottom touched, so no guard page is stepped over; the
                # fixed frame stays reached through rbp, and every epilogue resets rsp from it.
                # The touch keeps the word it reaches, which is the saved rbp when the count
                # is zero and the function keeps nothing in its frame
                let bytes = try self.source(n.a, ir.Class.l, "%rax")
                try self.writer.line3("subq ", bytes, ", %rsp")
                try self.writer.line("orq $0, (%rsp)")
                let d = self.target(n.dst, ir.Class.l, "%rax")
                try self.writer.line3("movq %rsp, ", d, "")
                try self.finish(n.dst, ir.Class.l, d)
            .frame:
                let d = self.target(n.dst, ir.Class.l, "%rax")
                if self.convention.windows:
                    try self.frame_address(d, 0)
                else:
                    try self.writer.line3("movq %rbp, ", d, "")
                try self.finish(n.dst, ir.Class.l, d)
            .extsb, .extub, .extsh, .extuh, .extsw, .extuw:
                let v = try self.source(n.a, ir.Class.w, "%eax")
                let d = self.target(n.dst, n.cls, self.reg(n.cls, 0))
                try self.extend_from(n.op, n.cls, v, d)
                try self.finish(n.dst, n.cls, d)
            .exth, .exts, .truncs, .truncd:
                try self.float_convert(n)
            .stosi, .stoui, .dtosi, .dtoui:
                try self.float_to_int(n)
            .swtof, .uwtof, .sltof, .ultof:
                try self.int_to_float(n)
            .cast:
                # bits move between an integer and a float of the same width
                let other = ir.cast_partner(n.cls)
                let v = try self.source(n.a, other, self.reg(other, 0))
                let d = self.target(n.dst, n.cls, self.reg(n.cls, 0))
                try self.writer.line4("movq " if n.cls == ir.Class.d or n.cls == ir.Class.l else "movd ", v, ", ", d)
                try self.finish(n.dst, n.cls, d)
            .addr:
                if self.fr.promoted_slot(n.dst) == 0:
                    let d = self.target(n.dst, ir.Class.l, "%rax")
                    try self.address_operand(d, n.a)
                    try self.finish(n.dst, ir.Class.l, d)
            .addr_tls:
                try object.thread_address(&self.object_format, &self.writer.out, n.a.name)
                try self.store_temp("%rax", ir.Class.l, n.dst)
            .blit:
                if n.extra <= regalloc.inline_block:
                    # from and to the addresses where they live
                    let origin = try self.source(n.b, ir.Class.l, "%r10")
                    let to = try self.source(n.a, ir.Class.l, "%r11")
                    try self.inline_copy(origin, to, (u64)n.extra)
                else:
                    # the origin first, through a scratch no value lives in: the destination's
                    # address may live in the second argument's register, and the origin's
                    # in the first's
                    try self.address_operand("%r10", n.b)
                    try self.address_operand(self.c_argument(0), n.a)
                    try self.writer.line3("movq %r10, ", self.c_argument(1), "")
                    try self.load_immediate(self.c_argument(2), (i64)n.extra)
                    try self.call_symbol("memcpy")
            .zero:
                if n.extra <= regalloc.inline_block:
                    let to = try self.source(n.a, ir.Class.l, "%r11")
                    try self.inline_zero(to, (u64)n.extra)
                else:
                    try self.address_operand(self.c_argument(0), n.a)
                    try self.load_immediate(narrow_name(self.c_argument(1)), 0)
                    try self.load_immediate(self.c_argument(2), (i64)n.extra)
                    try self.call_symbol("memset")
            .fill:
                try self.address_operand(self.c_argument(0), n.a)
                try self.load_immediate(narrow_name(self.c_argument(1)), n.b.value)
                try self.load_immediate(self.c_argument(2), (i64)n.extra)
                try self.call_symbol("memset")
            .call:
                try self.call(n)
            .word:
                # moved out by the call before it (`take_words`)
                _ = 0
            .reserve:
                try self.reserve_levels(n.extra)
            .trap_:
                if n.b.kind == ir.RefKind.text:
                    # the position first, then the reason (§11.5)
                    try self.address_operand(self.c_argument(0), n.b)
                    try self.address_operand(self.c_argument(1), n.a)
                    try self.call_symbol(self.program_unit.trap_at_function)
                else:
                    try self.address_operand(self.c_argument(0), n.a)
                    try self.call_symbol(self.program_unit.trap_function)
            .bounds:
                let a = try self.source(n.a, ir.Class.l, "%rax")
                let b = try self.source(n.b, ir.Class.l, "%r11")
                try self.writer.line4("cmpq ", b, ", ", a)
                try self.check_failure("jbe " if n.extra == 1 else "jb ", "ja " if n.extra == 1 else "jae ", "index out of bounds", n)
            .addo, .subo, .mulo:
                try self.checked(n)
            .asm_:
                try self.inline_asm(n)
            .set_reg:
                # move an input operand into the exact register the asm block names
                let reg = with_percent(self.program_unit.texts.get((usize)n.b.index).bytes)
                try self.operand(reg, n.cls, n.a)
            .get_reg:
                # the outputs of an asm block are stored by `inline_asm`, which follows the
                # block's operands on both sides
                _ = n.cls
            .atomic_load:
                # every aligned load is acquire on this target; the fence is for sequential
                # consistency against stores, which `atomic_store` supplies with `xchg`
                let cls = ir.Class.w if n.extra <= 4 else ir.Class.l
                try self.address_operand("%r10", n.a)
                try self.writer.line4(zero_extending_load(n.extra), "(%r10), ", self.reg(cls, 0), "")
                try self.store_temp(self.reg(cls, 0), cls, n.dst)
            .atomic_store:
                let cls = ir.Class.w if n.extra <= 4 else ir.Class.l
                try self.operand(self.reg(cls, 0), cls, n.a)
                try self.address_operand("%r10", n.b)
                try self.writer.line("xchg" if n.extra2 >= 5 else "mov")
                try self.writer.put(width_suffix(n.extra))
                try self.writer.put(" ")
                try self.writer.put(register_of_width("%rax", (u64)n.extra))
                try self.writer.put(", (%r10)")
            .atomic_add, .atomic_sub, .atomic_or, .atomic_and_not, .atomic_xor, .atomic_swap, .atomic_max, .atomic_min:
                try self.read_modify_write(n)
            .atomic_cas:
                try self.compare_and_swap(n)
            .fence:
                # a signal fence (6) orders the compiler alone: no instruction
                if n.a.value != 6:
                    try self.writer.line("mfence")
            .fma:
                try self.fused(n)
            .select:
                try self.selected(n)
            .loadv, .storev, .vadd, .vsub, .vmul, .vdiv, .vand, .vor, .vxor, .vneg, .vnot, .vaddw, .vsubw, .vmulw, .vaddo, .vsubo, .vadds, .vsubs, .vshl, .vshr, .vsplat:
                try self.vector(n)
            _:
                error(failed, "an instruction the x86_64 generator does not know")

    ## `d = a op b` on the integer classes: the value into `d`, then the two-operand form.
    func arithmetic(n: ir.Instr) -> !:
        let cls = n.cls
        let suffix = "q" if cls == ir.Class.l else "l"
        if n.op == ir.Op.sdiv or n.op == ir.Op.udiv or n.op == ir.Op.srem or n.op == ir.Op.urem:
            # the dividend in rax (and rdx), the divisor anywhere else: out of rdx, a
            # temporary's register the extension overwrites (`fixed_clobbers`), first
            var b = try self.source(n.b, cls, self.reg(cls, 1))
            if wide_name(b) == "%rdx":
                try self.writer.line4("mov", suffix, " ", b)
                try self.put2(", ", self.reg(cls, 1))
                b = self.reg(cls, 1)
            try self.operand(self.reg(cls, 0), cls, n.a)
            if n.op == ir.Op.sdiv or n.op == ir.Op.srem:
                try self.writer.line("cqto" if cls == ir.Class.l else "cltd")
                try self.writer.line3("idiv", suffix, " ")
            else:
                try self.writer.line("xorl %edx, %edx")
                try self.writer.line3("div", suffix, " ")
            try self.writer.put(b)
            let remainder = "%rdx" if cls == ir.Class.l else "%edx"
            let quotient = self.reg(cls, 0) if n.op == ir.Op.sdiv or n.op == ir.Op.udiv else remainder
            let d = self.target(n.dst, cls, self.reg(cls, 0))
            if d != quotient:
                try self.writer.line4("mov", suffix, " ", quotient)
                try self.writer.put(", ")
                try self.writer.put(d)
            try self.finish(n.dst, cls, d)
            return
        if n.b.kind == ir.RefKind.constant and n.b.value >= -2147483648 and n.b.value <= 2147483647:
            try self.arithmetic_immediate(n, suffix)
            return
        if n.op == ir.Op.shl or n.op == ir.Op.lshr or n.op == ir.Op.ashr:
            try self.variable_shift(n, suffix)
            return
        let a = try self.source(n.a, cls, self.reg(cls, 0))
        var b = try self.source(n.b, cls, self.reg(cls, 1))
        let d = self.target(n.dst, cls, self.reg(cls, 0))
        # A dying RHS may share the destination. Preserve it before installing the
        # LHS: x86's destructive two-operand form must not turn a+b into a+a.
        if b == d and a != d:
            try self.writer.line4("mov", suffix, " ", b)
            try self.writer.put(", ")
            try self.writer.put(self.reg(cls, 1))
            b = self.reg(cls, 1)
        if a != d:
            try self.writer.line4("mov", suffix, " ", a)
            try self.writer.put(", ")
            try self.writer.put(d)
        var mnemonic = "add"
        match n.op:
            .add: mnemonic = "add"
            .sub: mnemonic = "sub"
            .mul: mnemonic = "imul"
            .and_: mnemonic = "and"
            .or_: mnemonic = "or"
            .xor: mnemonic = "xor"
            _: mnemonic = "add"
        try self.writer.line4(mnemonic, suffix, " ", b)
        try self.writer.put(", ")
        try self.writer.put(d)
        try self.finish(n.dst, cls, d)

    ## A shift by a count in a register, which the instruction reads from cl: rcx, which no
    ## temporary keeps across it (`fixed_clobbers`), but the value, the count or the result
    ## may live in. The value is read out of rcx before the count goes in, and the shift
    ## happens in rax when the result's register is rcx.
    func variable_shift(n: ir.Instr, suffix: str) -> !:
        let cls = n.cls
        let count_reg = "%rcx" if cls == ir.Class.l else "%ecx"
        var a = try self.source(n.a, cls, self.reg(cls, 0))
        let c = try self.source(n.b, cls, self.reg(cls, 1))
        var d = self.target(n.dst, cls, self.reg(cls, 0))
        if d == count_reg:
            d = self.reg(cls, 0)
        if c != count_reg:
            if a == count_reg:
                try self.writer.line5("mov", suffix, " ", a, ", ")
                try self.writer.put(self.reg(cls, 2))
                a = self.reg(cls, 2)
            try self.writer.line5("mov", suffix, " ", c, ", ")
            try self.writer.put(count_reg)
        if a != d:
            try self.writer.line5("mov", suffix, " ", a, ", ")
            try self.writer.put(d)
        try self.writer.line("shl" if n.op == ir.Op.shl else ("shr" if n.op == ir.Op.lshr else "sar"))
        try self.writer.put(suffix)
        try self.writer.put(" %cl, ")
        try self.writer.put(d)
        try self.finish(n.dst, cls, d)

    ## `d = a op c` on the integer classes with a constant `c` that fits a sign-extended
    ## 32-bit immediate, carried by the instruction: a multiply by 3, 5 or 9 as one `lea`, by
    ## a power of two as a shift, by anything else as the three-operand `imul`; a shift by
    ## its count's low bits, which is what the instruction reads; the rest in the
    ## immediate form, after the value moved into `d`.
    func arithmetic_immediate(n: ir.Instr, suffix: str) -> !:
        let cls = n.cls
        let c = n.b.value
        let a = try self.source(n.a, cls, self.reg(cls, 0))
        let d = self.target(n.dst, cls, self.reg(cls, 0))
        var shift: i64 = -1
        if n.op == ir.Op.mul:
            if c == 3 or c == 5 or c == 9:
                try self.writer.line5("lea", suffix, " (", wide_name(a), ",")
                try self.writer.put(wide_name(a))
                try self.writer.put(",")
                try self.writer.put_i64(c - 1)
                try self.put2("), ", d)
                try self.finish(n.dst, cls, d)
                return
            if c > 1 and (c & (c - 1)) == 0:
                shift = 0
                while ((i64)1 << shift) != c:
                    shift += 1
            else:
                try self.writer.line3("imul", suffix, " $")
                try self.writer.put_i64(c)
                try self.put2(", ", a)
                try self.put2(", ", d)
                try self.finish(n.dst, cls, d)
                return
        if a != d:
            try self.writer.line5("mov", suffix, " ", a, ", ")
            try self.writer.put(d)
        var mnemonic = "add"
        match n.op:
            .add: mnemonic = "add"
            .sub: mnemonic = "sub"
            .and_: mnemonic = "and"
            .or_: mnemonic = "or"
            .xor: mnemonic = "xor"
            .shl, .mul: mnemonic = "shl"
            .lshr: mnemonic = "shr"
            _: mnemonic = "sar"
        let shifts = n.op == ir.Op.mul or n.op == ir.Op.shl or n.op == ir.Op.lshr or n.op == ir.Op.ashr
        if shifts and shift < 0:
            shift = c & (63 if cls == ir.Class.l else 31)
        try self.writer.line3(mnemonic, suffix, " $")
        try self.writer.put_i64(shift if shifts else c)
        try self.put2(", ", d)
        try self.finish(n.dst, cls, d)

    ## Float arithmetic: singles and doubles directly, into the result's own register when it
    ## has one, in the three-operand AVX form from x86-64-v3; halves through single precision
    ## in scratch registers.
    func float_arithmetic(given: ir.Instr) -> !:
        var n = given
        if (n.op == ir.Op.add or n.op == ir.Op.mul) and n.cls != ir.Class.h and self.pooled_constant(n.b) == none:
            if let bits = self.pooled_constant(n.a):
                # a constant on the left of a sum or a product goes right, where the
                # instruction reads it from the pool itself: the same value, as the
                # constant is no NaN whose payload the order would choose
                if not is_nan_bits(bits, n.cls):
                    n.a = given.b
                    n.b = given.a
        let cls = n.cls
        let a = try self.source(n.a, cls, self.reg(cls, 0))
        var b = self.reg(cls, 1)
        if cls == ir.Class.h or (self.frame_home(n.b) == none and self.pooled_constant(n.b) == none):
            b = try self.source(n.b, cls, b)
        var mnemonic = "add"
        match n.op:
            .add: mnemonic = "add"
            .sub: mnemonic = "sub"
            .mul: mnemonic = "mul"
            .sdiv: mnemonic = "div"
            _: mnemonic = "add"
        let form = "sd " if cls == ir.Class.d else "ss "
        if cls == ir.Class.h:
            let d = self.reg(cls, 0)
            if a != d:
                try self.writer.line4("movaps ", a, ", ", d)
            try self.widen_half(d)
            if b != self.reg(cls, 1):
                try self.writer.line4("movaps ", b, ", ", self.reg(cls, 1))
            try self.widen_half(self.reg(cls, 1))
            try self.writer.line5(mnemonic, form, self.reg(cls, 1), ", ", d)
            try self.round_to_half(d)
            try self.finish(n.dst, cls, d)
            return
        let d = self.target(n.dst, cls, self.reg(cls, 0))
        # a right operand kept in the frame, or a constant in the literal pool, is read
        # there, by the instruction itself
        let home = self.frame_home(n.b)
        let pooled = self.pooled_constant(n.b)
        if self.level >= 3:
            try self.writer.line3("v", mnemonic, form)
            if let below = home:
                try self.frame_operand(below)
            elif let bits = pooled:
                try self.pooled(bits)
            else:
                try self.writer.put(b)
            try self.put2(", ", a)
            try self.put2(", ", d)
            try self.finish(n.dst, cls, d)
            return
        if home != none or pooled != none:
            if a != d:
                try self.writer.line4("movaps ", a, ", ", d)
            try self.writer.line2(mnemonic, form)
            if let below = home:
                try self.frame_operand(below)
            elif let bits = pooled:
                try self.pooled(bits)
            try self.put2(", ", d)
            try self.finish(n.dst, cls, d)
            return
        # the destructive form must not overwrite a right operand that shares the result's
        # register before reading it: `a - b` is not `a - a`
        if b == d and a != d:
            try self.writer.line4("movaps ", b, ", ", self.reg(cls, 1))
            b = self.reg(cls, 1)
        if a != d:
            try self.writer.line4("movaps ", a, ", ", d)
        try self.writer.line5(mnemonic, form, b, ", ", d)
        try self.finish(n.dst, cls, d)

    ## The single in `reg` rounded to a half in its low sixteen bits, zero above, so that a
    ## reinterpretation of the bits as an integer reads the half alone: F16C from
    ## x86-64-v3, the unit's own routine below it.
    func round_to_half(reg: str) -> !:
        try self.writer.line3("movd ", reg, ", %eax")
        if self.level >= 3:
            try self.writer.line4("vcvtps2ph $0, ", reg, ", ", reg)
            try self.writer.line3("movd ", reg, ", %eax")
            try self.writer.line("movzwl %ax, %eax")
        else:
            self.half_routines = true
            try self.writer.line2("call ", self.shared(".Llb_single_to_half"))
        try self.writer.line3("movd %eax, ", reg, "")

    ## The half in the low sixteen bits of `reg` widened to a single, in place.
    func widen_half(reg: str) -> !:
        if self.level >= 3:
            try self.writer.line4("vcvtph2ps ", reg, ", ", reg)
            return
        self.half_routines = true
        try self.writer.line3("movd ", reg, ", %eax")
        try self.writer.line2("call ", self.shared(".Llb_half_to_single"))
        try self.writer.line3("movd %eax, ", reg, "")

    func negate(n: ir.Instr) -> !:
        let cls = n.cls
        let v = try self.source(n.a, cls, self.reg(cls, 0))
        let d = self.target(n.dst, cls, self.reg(cls, 0))
        if cls == ir.Class.w or cls == ir.Class.l:
            if v != d:
                try self.writer.line4("movq " if cls == ir.Class.l else "movl ", v, ", ", d)
            try self.writer.line3("negq " if cls == ir.Class.l else "negl ", d, "")
        elif cls == ir.Class.d:
            try self.writer.line3("movq ", v, ", %rax")
            try self.writer.line("btcq $63, %rax")
            try self.writer.line3("movq %rax, ", d, "")
        else:
            try self.writer.line3("movd ", v, ", %eax")
            try self.writer.line("xorl $32768, %eax" if cls == ir.Class.h else "xorl $2147483648, %eax")
            try self.writer.line3("movd %eax, ", d, "")
        try self.finish(n.dst, cls, d)

    ## The C library's float functions this target computes itself (`isa.has_float`): the
    ## square root, the magnitude (the sign bit cleared, a NaN's payload kept), and from
    ## x86-64-v2 SSE4.1's `roundss` toward negative infinity (mode 9), positive infinity (10)
    ## or zero (11), bit 3 suppressing the inexact exception as C's `floor` does.
    func float_function(n: ir.Instr) -> !:
        let cls = n.cls
        let double = cls == ir.Class.d
        let v = try self.source(n.a, cls, self.reg(cls, 0))
        let d = self.target(n.dst, cls, self.reg(cls, 0))
        match n.op:
            .fabs:
                if double:
                    try self.writer.line3("movq ", v, ", %rax")
                    try self.writer.line("btrq $63, %rax")
                    try self.writer.line3("movq %rax, ", d, "")
                else:
                    try self.writer.line3("movd ", v, ", %eax")
                    try self.writer.line("andl $2147483647, %eax")
                    try self.writer.line3("movd %eax, ", d, "")
            .fsqrt:
                try self.writer.line4("sqrtsd " if double else "sqrtss ", v, ", ", d)
            _:
                let mode = "$9, " if n.op == ir.Op.ffloor else ("$10, " if n.op == ir.Op.fceil else "$11, ")
                try self.writer.line4("roundsd " if double else "roundss ", mode, v, ", ")
                try self.writer.put(d)
        try self.finish(n.dst, cls, d)

    ## `switch`: the value less the table's first is the entry, when it is below the
    ## table's count; the entry is its label's distance from the table, which follows the
    ## jump in the text, so the table needs no relocation. rax holds the entry, r11 the
    ## table's address.
    func jump_table(n: ir.Instr) -> !:
        let wide = n.cls == ir.Class.l
        let table = self.f.jump_tables.get((usize)n.extra2)
        let index = "%rax" if wide else "%eax"
        try self.operand(index, n.cls, n.a)
        if wide and table.low != 0:
            if table.low >= -2147483648 and table.low <= 2147483647:
                try self.writer.line("subq $")
                try self.writer.put_i64(table.low)
                try self.writer.put(", %rax")
            else:
                try self.load_immediate("%r11", table.low)
                try self.writer.line("subq %r11, %rax")
        elif not wide:
            # the low bits are the whole value at the word's width, spelled signed
            let low = (i64)((u64)table.low & 0xFFFFFFFF)
            if low != 0:
                try self.writer.line("subl $")
                try self.writer.put_i64(low - 4294967296 if low > 2147483647 else low)
                try self.writer.put(", %eax")
        try self.writer.line("cmpq $" if wide else "cmpl $")
        try self.writer.put_u64((u64)table.count)
        try self.writer.put(", ")
        try self.writer.put(index)
        try self.writer.line("jae ")
        try self.label_name(n.extra)
        try self.writer.line("leaq ")
        try self.table_name(n.extra2)
        try self.writer.put("(%rip), %r11")
        # a word's subtraction left the high half of rax zero
        try self.writer.line("movslq (%r11,%rax,4), %rax")
        try self.writer.line("addq %r11, %rax")
        try self.writer.line("jmp *%rax")
        try self.writer.line(".p2align 2")
        try self.writer.put("\n")
        try self.table_name(n.extra2)
        try self.writer.put(":")
        for label in self.f.table_targets(n.extra2):
            try self.writer.line(".long ")
            try self.label_name(label)
            try self.writer.put(" - ")
            try self.table_name(n.extra2)

    func comparison(n: ir.Instr, branch: ir.Instr? = none, next_label: u32 = 0) -> !:
        let cls = n.cls
        let a = try self.source(n.a, cls, self.reg(cls, 0))
        # a constant the instruction carries: a sign-extended 32-bit immediate
        if n.b.kind == ir.RefKind.constant and n.b.value >= -2147483648 and n.b.value <= 2147483647:
            try self.writer.line("cmpq $" if cls == ir.Class.l else "cmpl $")
            try self.writer.put_i64(n.b.value)
            try self.put2(", ", a)
        else:
            let b = try self.source(n.b, cls, self.reg(cls, 1))
            try self.writer.line4("cmpq " if cls == ir.Class.l else "cmpl ", b, ", ", a)
        let cond = condition_of(n.op)
        if let jump = branch:
            try self.writer.line("j")
            try self.writer.put(cond)
            try self.writer.put(" ")
            try self.label_name(jump.extra2 if next_label == jump.extra else jump.extra)
            if next_label != jump.extra and next_label != jump.extra2:
                try self.writer.line("jmp ")
                try self.label_name(jump.extra2)
            return
        try self.writer.line3("set", cond, " %al")
        let d = self.target(n.dst, ir.Class.w, "%eax")
        try self.writer.line3("movzbl %al, ", d, "")
        try self.finish(n.dst, ir.Class.w, d)

    ## Float comparison: `ucomis` orders the second operand against the first and leaves the
    ## unordered case with every flag set, so `<` and `<=` swap operands to use the
    ## above-conditions, which an unordered pair fails, and equality also tests parity.
    func float_comparison(n: ir.Instr) -> !:
        let cls = n.cls
        var a = try self.source(n.a, cls, self.reg(cls, 0))
        var b = try self.source(n.b, cls, self.reg(cls, 1))
        if cls == ir.Class.h:
            if a != self.reg(cls, 0):
                try self.writer.line4("movaps ", a, ", ", self.reg(cls, 0))
            if b != self.reg(cls, 1):
                try self.writer.line4("movaps ", b, ", ", self.reg(cls, 1))
            a = self.reg(cls, 0)
            b = self.reg(cls, 1)
            try self.widen_half(a)
            try self.widen_half(b)
        let mnemonic = "ucomisd " if cls == ir.Class.d else "ucomiss "
        match n.op:
            .cslt:
                try self.writer.line4(mnemonic, a, ", ", b)
                try self.writer.line("seta %al")
            .csle:
                try self.writer.line4(mnemonic, a, ", ", b)
                try self.writer.line("setae %al")
            .csgt:
                try self.writer.line4(mnemonic, b, ", ", a)
                try self.writer.line("seta %al")
            .csge:
                try self.writer.line4(mnemonic, b, ", ", a)
                try self.writer.line("setae %al")
            .ceq:
                try self.writer.line4(mnemonic, b, ", ", a)
                try self.writer.line("sete %al")
                try self.writer.line("setnp %r11b")
                try self.writer.line("andb %r11b, %al")
            _:
                try self.writer.line4(mnemonic, b, ", ", a)
                try self.writer.line("setne %al")
                try self.writer.line("setp %r11b")
                try self.writer.line("orb %r11b, %al")
        let d = self.target(n.dst, ir.Class.w, "%eax")
        try self.writer.line3("movzbl %al, ", d, "")
        try self.finish(n.dst, ir.Class.w, d)

    ## `op` applied to the 32-bit view `v` of a value, into `d` of class `cls`.
    func extend_from(op: ir.Op, cls: ir.Class, v: str, d: str) -> !:
        let to = "q " if cls == ir.Class.l else "l "
        match op:
            .extsb: try self.writer.line4("movsb", to, byte_name(v), ", ")
            .extub: try self.writer.line4("movzb", to, byte_name(v), ", ")
            .extsh: try self.writer.line4("movsw", to, word_name(v), ", ")
            .extuh: try self.writer.line4("movzw", to, word_name(v), ", ")
            .extsw:
                if cls == ir.Class.l:
                    try self.writer.line3("movslq ", v, ", ")
                else:
                    try self.writer.line3("movl ", v, ", ")
            _:
                # zero-extension of a word: writing the 32-bit view clears the upper half
                try self.writer.line3("movl ", v, ", ")
                try self.writer.put(narrow_name(d))
                return
        try self.writer.put(d)

    ## Conversions between float widths: `exth` a half up, `exts` a single up, `truncs` a
    ## single down to a half, `truncd` a double down.
    func float_convert(n: ir.Instr) -> !:
        let src = ir.float_source_class(n.op)
        let v = try self.source(n.a, src, self.reg(src, 0))
        # a single and a double convert straight into the result's register; a half's
        # widening and rounding work in scratch
        let direct = n.op == ir.Op.exts or (n.op == ir.Op.truncd and n.cls != ir.Class.h)
        let d = self.target(n.dst, n.cls, self.reg(n.cls, 0)) if direct else self.reg(n.cls, 0)
        match n.op:
            .exth:
                if v != d:
                    try self.writer.line4("movaps ", v, ", ", d)
                try self.widen_half(d)
                if n.cls == ir.Class.d:
                    try self.writer.line4("cvtss2sd ", d, ", ", d)
            .exts:
                try self.writer.line4("cvtss2sd ", v, ", ", d)
            .truncs:
                if v != d:
                    try self.writer.line4("movaps ", v, ", ", d)
                try self.round_to_half(d)
            _:
                try self.writer.line4("cvtsd2ss ", v, ", ", d)
                if n.cls == ir.Class.h:
                    try self.round_to_half(d)
        try self.finish(n.dst, n.cls, d)

    ## A float to an integer, truncating: signed through the conversion instruction; an
    ## unsigned word through the 64-bit conversion; an unsigned long by taking 2^63 off
    ## first when the value reaches it. A saturating signed conversion (`extra` 1) fixes
    ## up what the instruction leaves out of range (`saturate`).
    func float_to_int(n: ir.Instr) -> !:
        let src = ir.Class.s if n.op == ir.Op.stosi or n.op == ir.Op.stoui else ir.Class.d
        let v = try self.source(n.a, src, self.reg(src, 0))
        let d = self.target(n.dst, n.cls, self.reg(n.cls, 0))
        let convert = "cvttsd2si " if src == ir.Class.d else "cvttss2si "
        if n.op == ir.Op.stosi or n.op == ir.Op.dtosi:
            try self.writer.line4(convert, v, ", ", d)
            if n.extra == 1:
                try self.saturate(n, src, v, d)
        elif n.cls == ir.Class.w:
            try self.writer.line4(convert, v, ", ", wide_name(d))
        else:
            let work = self.reg(src, 0)
            let bound = self.reg(src, 1)
            if v != work:
                try self.writer.line4("movaps ", v, ", ", work)
            if src == ir.Class.d:
                try self.load_immediate("%rax", 4890909195324358656)
                try self.writer.line3("movq %rax, ", bound, "")
                try self.writer.line4("ucomisd ", bound, ", ", work)
            else:
                try self.load_immediate("%eax", 1593835520)
                try self.writer.line3("movd %eax, ", bound, "")
                try self.writer.line4("ucomiss ", bound, ", ", work)
            try self.writer.line("jae 1f")
            try self.writer.line4(convert, work, ", ", d)
            try self.writer.line("jmp 2f")
            try self.writer.put("\n1:")
            try self.writer.line4("subsd " if src == ir.Class.d else "subss ", bound, ", ", work)
            try self.writer.line4(convert, work, ", ", d)
            try self.writer.line3("btcq $63, ", d, "")
            try self.writer.put("\n2:")
        try self.finish(n.dst, n.cls, d)

    ## After `cvttsd2si`, which leaves the least integer for NaN and for every value out of
    ## range: the greatest for a value at or above 2^(bits-1), 0 for NaN, as arm64's
    ## `fcvtzs` and `core.f_to_s` answer (the least is already right below the range).
    func saturate(n: ir.Instr, src: ir.Class, v: str, d: str) -> !:
        let wide = n.cls == ir.Class.l
        let bound = self.reg(src, 1)
        if src == ir.Class.d:
            try self.load_immediate("%r11", 4890909195324358656 if wide else 4746794007248502784)
            try self.writer.line3("movq %r11, ", bound, "")
            try self.writer.line4("ucomisd ", bound, ", ", v)
        else:
            try self.load_immediate("%r11d", 1593835520 if wide else 1325400064)
            try self.writer.line3("movd %r11d, ", bound, "")
            try self.writer.line4("ucomiss ", bound, ", ", v)
        # below the bound, or unordered
        try self.writer.line("jb 1f")
        try self.load_immediate(d, 9223372036854775807 if wide else 2147483647)
        try self.writer.put("\n1:")
        try self.writer.line4("ucomisd " if src == ir.Class.d else "ucomiss ", v, ", ", v)
        try self.writer.line("jnp 2f")
        try self.load_immediate(d, 0)
        try self.writer.put("\n2:")

    ## An integer to a float: signed through the conversion instruction at the integer's
    ## width; an unsigned word as a long; an unsigned long halved when its top bit is set,
    ## keeping the low bit for the rounding, then doubled.
    func int_to_float(n: ir.Instr) -> !:
        let src = ir.Class.w if n.op == ir.Op.swtof or n.op == ir.Op.uwtof else ir.Class.l
        let v = try self.source(n.a, src, self.reg(src, 0))
        let d = self.target(n.dst, n.cls, self.reg(n.cls, 0)) if n.cls != ir.Class.h else self.reg(n.cls, 0)
        let single = n.cls != ir.Class.d
        let convert = "cvtsi2ss" if single else "cvtsi2sd"
        if n.op == ir.Op.swtof:
            try self.writer.line4(convert, "l ", v, ", ")
            try self.writer.put(d)
        elif n.op == ir.Op.sltof:
            try self.writer.line4(convert, "q ", v, ", ")
            try self.writer.put(d)
        elif n.op == ir.Op.uwtof:
            try self.writer.line3("movl ", v, ", %eax")
            try self.writer.line3(convert, "q %rax, ", d)
        else:
            try self.writer.line3("movq ", v, ", %r11")
            try self.writer.line("testq %r11, %r11")
            try self.writer.line("js 1f")
            try self.writer.line3(convert, "q %r11, ", d)
            try self.writer.line("jmp 2f")
            try self.writer.put("\n1:")
            try self.writer.line("movq %r11, %rax")
            try self.writer.line("shrq $1, %rax")
            try self.writer.line("andl $1, %r11d")
            try self.writer.line("orq %r11, %rax")
            try self.writer.line3(convert, "q %rax, ", d)
            try self.writer.line4("addss " if single else "addsd ", d, ", ", d)
            try self.writer.put("\n2:")
        if n.cls == ir.Class.h:
            try self.round_to_half(d)
        try self.finish(n.dst, n.cls, d)

    ## `fma` (x86-64-v3, `isa.has_fma`): FMA3 accumulates into its destination, so the addend
    ## goes to the third scratch register and `vfmadd231` adds the product of the other two
    ## to it. Of two NaN operands the instruction returns its second operand's, then its
    ## third's, so `b` goes second and `a` third: `b`'s NaN before `a`'s, the one the C
    ## libraries' `fma` returns.
    func fused(n: ir.Instr) -> !:
        let addend = self.f.args.get((usize)n.extra2).value
        let sum = self.reg(n.cls, 2)
        let a = try self.source(n.a, n.cls, self.reg(n.cls, 0))
        let b = try self.source(n.b, n.cls, self.reg(n.cls, 1))
        let c = try self.source(addend, n.cls, sum)
        if c != sum:
            try self.writer.line4("movaps ", c, ", ", sum)
        let wide = n.cls == ir.Class.d or (n.cls == ir.Class.v and ir.lane_bits(n.extra) == 64)
        let form = ("pd" if wide else "ps") if n.cls == ir.Class.v else ("sd" if wide else "ss")
        try self.writer.line4("vfmadd231", form, " ", a)
        try self.writer.put(", ")
        try self.writer.put(b)
        try self.writer.put(", ")
        try self.writer.put(sum)
        let d = self.target(n.dst, n.cls, sum)
        if d != sum:
            try self.writer.line4("movaps ", sum, ", ", d)
        try self.finish(n.dst, n.cls, d)

    ## Checked add, subtract, and multiply: trap on overflow of the class width. A checked
    ## operation reads its right operand where it lives and computes into the
    ## destination's register when it has one, its left operand moved there first; an
    ## unsigned multiply computes in rax as the instruction demands.
    func checked(n: ir.Instr) -> !:
        let signed = n.extra == 1
        let cls = n.cls
        let suffix = "q" if cls == ir.Class.l else "l"
        if n.op == ir.Op.mulo and not signed and constant_factor(n.b):
            try self.checked_constant(n, suffix)
            return
        if n.op == ir.Op.mulo and not signed:
            let a = self.reg(cls, 0)
            try self.operand(a, cls, n.a)
            let b = try self.source(n.b, cls, self.reg(cls, 1))
            try self.writer.line4("mul", suffix, " ", b)
            try self.check_failure("jnc ", "jc ", "integer overflow", n)
            try self.store_temp(a, cls, n.dst)
            return
        if n.b.kind == ir.RefKind.constant and n.b.value >= -2147483648 and n.b.value <= 2147483647:
            try self.checked_immediate(n, suffix)
            return
        let b = try self.source(n.b, cls, self.reg(cls, 1))
        var d = self.target(n.dst, cls, self.reg(cls, 0))
        if d == b:
            # the destination is the right operand's register: compute in scratch
            d = self.reg(cls, 0)
        let a = try self.source(n.a, cls, d)
        if a != d:
            try self.writer.line3(move_of(cls), a, ", ")
            try self.writer.put(d)
        if n.op == ir.Op.mulo:
            try self.writer.line5("imul", suffix, " ", b, ", ")
            try self.writer.put(d)
            try self.check_failure("jno ", "jo ", "integer overflow", n)
        else:
            try self.writer.line5("add" if n.op == ir.Op.addo else "sub", suffix, " ", b, ", ")
            try self.writer.put(d)
            if signed:
                try self.check_failure("jno ", "jo ", "integer overflow", n)
            else:
                try self.check_failure("jnc ", "jc ", "integer overflow", n)
        try self.finish(n.dst, cls, d)

    ## A checked operation of `checked` whose right operand is a constant that fits a
    ## sign-extended 32-bit immediate: a signed multiply in the three-operand `imul`, which
    ## sets the overflow flag as the two-operand form does; a sum or a difference in the
    ## immediate form, after the left operand moved into `d`.
    func checked_immediate(n: ir.Instr, suffix: str) -> !:
        let cls = n.cls
        let signed = n.extra == 1
        let d = self.target(n.dst, cls, self.reg(cls, 0))
        let a = try self.source(n.a, cls, d)
        if n.op == ir.Op.mulo:
            try self.writer.line3("imul", suffix, " $")
            try self.writer.put_i64(n.b.value)
            try self.put2(", ", a)
            try self.put2(", ", d)
            try self.check_failure("jno ", "jo ", "integer overflow", n)
            try self.finish(n.dst, cls, d)
            return
        if a != d:
            try self.writer.line3(move_of(cls), a, ", ")
            try self.writer.put(d)
        try self.writer.line3("add" if n.op == ir.Op.addo else "sub", suffix, " $")
        try self.writer.put_i64(n.b.value)
        try self.put2(", ", d)
        if signed:
            try self.check_failure("jno ", "jo ", "integer overflow", n)
        else:
            try self.check_failure("jnc ", "jc ", "integer overflow", n)
        try self.finish(n.dst, cls, d)

    ## An unsigned checked multiply by a constant `c` of `constant_factor`: the product
    ## overflows exactly when the factor is above the class's greatest value divided by `c`,
    ## which one comparison decides; the product is then the low half the three-operand
    ## `imul` (or `lea` for 3, 5 and 9) leaves, with no rdx:rax.
    func checked_constant(n: ir.Instr, suffix: str) -> !:
        let cls = n.cls
        let c = n.b.value
        let d = self.target(n.dst, cls, self.reg(cls, 0))
        let a = try self.source(n.a, cls, d)
        let limit = (i64)(0xFFFFFFFFFFFFFFFF // (u64)c) if cls == ir.Class.l else (i64)(0xFFFFFFFF // (u64)c)
        if limit <= 2147483647:
            try self.writer.line3("cmp", suffix, " $")
            try self.writer.put_i64(limit)
            try self.put2(", ", a)
        else:
            try self.load_immediate("%r11", limit)
            try self.writer.line3("cmpq %r11, ", a, "")
        try self.check_failure("jbe ", "ja ", "integer overflow", n)
        if c == 3 or c == 5 or c == 9:
            try self.writer.line5("lea", suffix, " (", wide_name(a), ",")
            try self.writer.put(wide_name(a))
            try self.writer.put(",")
            try self.writer.put_i64(c - 1)
            try self.put2("), ", d)
        else:
            try self.writer.line3("imul", suffix, " $")
            try self.writer.put_i64(c)
            try self.put2(", ", a)
            try self.put2(", ", d)
        try self.finish(n.dst, cls, d)

    ## Where a check of `n` that fails goes, after the comparison that decides it: with
    ## `--debug` to its trap inline, a branch on `pass` jumping over it; otherwise a branch
    ## on `fail` to a path after the function's body (`finish_cold`), so the passing path
    ## falls through and a loop's body holds no trap's instructions. `pass` and `fail` are
    ## the branch before its label (`jb `).
    func check_failure(pass: str, fail: str, reason: str, n: ir.Instr) -> !:
        if self.f.debug:
            try self.writer.line2(pass, "1f")
            try self.trap_text(reason, n.extra2)
            try self.writer.put("\n1:")
            return
        self.cold_count += 1
        try self.writer.line(fail)
        try self.cold_label(self.cold_count)
        # the trap goes to the cold paths' text, through the writer every emission uses
        let body = self.writer.out
        self.writer.out = self.cold
        try self.writer.put("\n")
        try self.cold_label(self.cold_count)
        try self.writer.put(":")
        if let dw = self.dw:
            try dw.line_of(&self.writer.out, self.f, n)
        try self.trap_text(reason, n.extra2)
        self.cold = self.writer.out
        self.writer.out = body

    ## The function's cold paths (`check_failure`), after its body and before its end.
    func finish_cold() -> !:
        if self.cold.used > 0:
            try self.writer.put(self.cold.view())
            self.cold.clear()
        self.cold_count = 0

    ## A trap with a fixed reason; `at`, when not zero, is the unit's text of the statement's
    ## position plus one, which the reporter names first (§11.5).
    func trap_text(reason: str, at: u32) -> !:
        # the reasons were added before the functions were generated (`trap_reasons`), so
        # the threads generating them only read the unit's texts
        let index = self.program_unit.find_text(reason) else error(ir.missing_text, "a trap's reason is not among the unit's texts")
        if at > 0:
            try self.address_operand(self.c_argument(0), ir.text_ref(at - 1))
            try self.address_operand(self.c_argument(1), ir.text_ref(index))
            try self.call_symbol(self.program_unit.trap_at_function)
            return
        try self.address_operand(self.c_argument(0), ir.text_ref(index))
        try self.call_symbol(self.program_unit.trap_function)