diff --git a/docs/NATIVE-INT.md b/docs/NATIVE-INT.md index a3ba09ecb8..5e635f7655 100644 --- a/docs/NATIVE-INT.md +++ b/docs/NATIVE-INT.md @@ -42,7 +42,9 @@ EL0, AArch64, little-endian data, no single step, no BTI, no MTE: |---|---| | data processing | ADD/SUB(S) immediate, shifted and extended register; AND/ORR/EOR/ANDS (immediate) and the shifted-register logical forms; MOVZ/MOVN/MOVK; SBFM/UBFM/BFM; EXTR; MADD/MSUB/SMADDL/SMSUBL/UMADDL/UMSUBL/SMULH/UMULH; UDIV/SDIV; LSLV/LSRV/ASRV/RORV; RBIT/REV16/REV32/REV/CLZ/CLS; ADC/SBC(S); CCMP/CCMN; CSEL/CSINC/CSINV/CSNEG | | PC-relative | ADR, ADRP | -| loads/stores | LDR/STR of general-purpose registers: unsigned and unscaled offsets, pre/post-index, register offset, sign-extending loads; LDP/STP/LDPSW | +| loads/stores | LDR/STR of general-purpose registers: unsigned and unscaled offsets, pre/post-index, register offset, sign-extending loads; LDP/STP/LDPSW; LDR (literal) | +| SIMD&FP loads/stores | LDR/STR of B/H/S/D/Q registers (same addressing modes), LDP/STP of S/D/Q registers, LDR (literal); no SVE/SME. The data goes through the CPU state (vector registers are not mapped) | +| system registers | MRS of plain 64-bit fields with no access check (TPIDR_EL0, TPIDRRO_EL0) | | branches | B.cond (when the guest flags are already in the host flags), CBZ/CBNZ, TBZ/TBNZ (inside a region) | Everything else stays on the existing paths, in particular system registers, diff --git a/hw/arm/tests/native_int_check.S b/hw/arm/tests/native_int_check.S index bdc584b489..78ab1e893e 100644 --- a/hw/arm/tests/native_int_check.S +++ b/hw/arm/tests/native_int_check.S @@ -13,6 +13,9 @@ * 4 x18, x29 and x30 as ordinary operands and load destinations * 5 NZCV set before a page-crossing (slow-path) load and store and * consumed after them + * 8 SIMD&FP loads in a region: a Q load and a pair that cross a 16 KiB + * page (slow path), and a Q load that faults: the destination stays + * unchanged and the registers written before it are exact * 7 MRS TPIDR_EL0 / TPIDRRO_EL0 (plain field reads) in the middle of * ALU code, after MSR TPIDR_EL0 at EL0 and TPIDRRO_EL0 set at EL1 * 6 a store over code later in the same block (the store's slow @@ -70,6 +73,9 @@ .text .globl _start _start: + mrs x0, cpacr_el1 + orr x0, x0, #(3 << 20) /* FPEN: SIMD&FP at EL0 */ + msr cpacr_el1, x0 adr x0, vectors msr vbar_el1, x0 ldr x0, =EL1_SP @@ -307,6 +313,32 @@ fault2: CHK x15, 0x1111 - 0x2222, 6 CHK x17, 1, 6 + /* 8: SIMD&FP loads and stores */ + LI x0, BUF + 0x4000 - 8 /* 16 bytes across a 16 KiB page */ + LI x1, 0x1111222233334444 + LI x2, 0x5555666677778888 + stp x1, x2, [x0] + ldr q0, [x0] + ldp d1, d2, [x0] + str q0, [x0, #-64] + ldp x3, x4, [x0, #-64] + fmov x5, d1 + fmov x6, d2 + CHK x3, 0x1111222233334444, 8 + CHK x4, 0x5555666677778888, 8 + CHK x5, 0x1111222233334444, 8 + CHK x6, 0x5555666677778888, 8 + movi v3.2d, #0xffffffffffffffff + LI x9, HOLE + 0x200 + movz x10, #0x2a + add x11, x10, #1 +fault8: + ldr q3, [x9] + CHKSAVE 11, 0x2b, 8 + CHKSAVE 31, HOLE + 0x200, 8 + fmov x12, d3 + CHK x12, 0xffffffffffffffff, 8 /* destination unchanged */ + /* 7: TLS register reads in the middle of a block */ LI x0, 0x0123456789abcdef msr tpidr_el0, x0 diff --git a/hw/arm/tests/native_int_gen.py b/hw/arm/tests/native_int_gen.py index 5f9a154d68..9123b20de9 100644 --- a/hw/arm/tests/native_int_gen.py +++ b/hw/arm/tests/native_int_gen.py @@ -25,6 +25,9 @@ import sys POOL = list(range(0, 18)) + [19, 20, 21] +# SIMD&FP registers the program may use: V8-V15 are callee-saved on the host (the harness). +VPOOL = list(range(0, 8)) + list(range(16, 32)) +VSZ = {0: "b", 1: "h", 2: "s", 3: "d", 4: "q"} CONDS = ["eq", "ne", "cs", "cc", "mi", "pl", "vs", "vc", "hi", "ls", "ge", "lt", "gt", "le", "al"] # buffer windows: base register, window start (offset in the 64 KiB buffer), initial offset in the window. # x27 sits 64 bytes below a 16 KiB page boundary, so its accesses cross pages (the TLB slow path). @@ -173,9 +176,86 @@ def window(self, b): start = BASES[b][0] return start, start + WINDOW + def vldst(self, wb=True): + """SIMD&FP loads and stores (and literal loads), like ldst().""" + r = self.r + b = r.choice(list(BASES)) + cur = self.off[b] + k = r.randrange(8) + v = lambda: r.choice(VPOOL) # noqa: E731 + if k <= 2: + size = r.randrange(5) + op = r.choice(["ldr", "str"]) + mode = r.choice(["uoff", "unscaled", "pre", "post"] if wb else ["uoff", "unscaled"]) + reg = f"{VSZ[size]}{v()}" + if mode == "uoff": + off = r.randrange(0, 128) << size + if cur + off + (1 << size) > WINDOW: + off = 0 + self.emit(f"{op} {reg}, [x{b}, #{off}]") + elif mode == "unscaled": + off = r.randrange(-256, 256) + if not (0 <= cur + off and cur + off + 16 <= WINDOW): + off = 0 + self.emit(f"{op.replace('ldr', 'ldur').replace('str', 'stur')} {reg}, [x{b}, #{off}]") + else: + imm = r.randrange(-256, 256) + lo, hi = 64, WINDOW - 64 + if not (lo <= cur + imm <= hi): + imm = -imm if lo <= cur - imm <= hi and -imm != 256 else 0 + self.emit(f"{op} {reg}, [x{b}, #{imm}]!" if mode == "pre" else f"{op} {reg}, [x{b}], #{imm}") + self.off[b] = cur + imm + elif k <= 3: + size = r.randrange(5) + op = r.choice(["ldr", "str"]) + self.emit(f"and x25, {self.x()}, #0xf8") + ext = r.choice(["lsl", "uxtw", "sxtw"]) + idx = "x25" if ext == "lsl" else "w25" + sh = r.choice([0, size]) if size else 0 + if ext == "lsl" and not sh: + self.emit(f"{op} {VSZ[size]}{v()}, [x{b}, {idx}]") + else: + self.emit(f"{op} {VSZ[size]}{v()}, [x{b}, {idx}, {ext} #{sh}]") + elif k <= 5: + size = r.randrange(2, 5) + op = r.choice(["ldp", "stp"]) + a_, c_ = v(), v() + while op == "ldp" and c_ == a_: + c_ = v() + scale = 1 << size + mode = r.choice(["off", "pre", "post"] if wb else ["off"]) + imm = r.randrange(-64, 64) * scale + lo, hi = 128, WINDOW - 128 + if not (lo <= cur + imm <= hi): + imm = -imm if lo <= cur - imm <= hi and -imm != 64 * scale else 0 + regs = f"{VSZ[size]}{a_}, {VSZ[size]}{c_}" + if mode == "off": + self.emit(f"{op} {regs}, [x{b}, #{imm}]") + elif mode == "pre": + self.emit(f"{op} {regs}, [x{b}, #{imm}]!") + self.off[b] = cur + imm + else: + self.emit(f"{op} {regs}, [x{b}], #{imm}") + self.off[b] = cur + imm + else: + # literal loads from a pool jumped over + self.label += 1 + n = self.label + if r.random() < 0.5: + self.emit(f"ldr {VSZ[r.randrange(2, 5)]}{v()}, 8{n}f") + else: + op, dst = r.choice([("ldr", self.x()), ("ldr", self.w()), ("ldrsw", self.x())]) + self.emit(f"{op} {dst}, 8{n}f") + self.emit(f"b 9{n}f") + self.out.append(f"8{n}:") + self.emit(f".quad {r.getrandbits(64):#x}, {r.getrandbits(64):#x}") + self.out.append(f"9{n}:") + def ldst(self, wb=True): """wb=False: no pre/post-index forms (inside a branch shadow the tracked base offsets must not move).""" r = self.r + if r.random() < 0.3: + return self.vldst(wb) b = r.choice(list(BASES)) cur = self.off[b] k = r.randrange(10) @@ -286,16 +366,20 @@ def other(self): def branch(self): r = self.r self.label += 1 + n = self.label k = r.randrange(3) if k == 0: - self.emit(f"b.{r.choice(CONDS[:-1])} 9{self.label}f") + self.emit(f"b.{r.choice(CONDS[:-1])} 9{n}f") elif k == 1: - self.emit(f"{r.choice(['cbz', 'cbnz'])} {self.rr(r.random() < 0.5)}, 9{self.label}f") + self.emit(f"{r.choice(['cbz', 'cbnz'])} {self.rr(r.random() < 0.5)}, 9{n}f") else: - self.emit(f"{r.choice(['tbz', 'tbnz'])} {self.x()}, #{r.randrange(64)}, 9{self.label}f") + self.emit(f"{r.choice(['tbz', 'tbnz'])} {self.x()}, #{r.randrange(64)}, 9{n}f") for _ in range(r.randint(0, 3)): - self.alu() if r.random() < 0.7 else self.ldst(wb=False) - self.out.append(f"9{self.label}:") + if r.random() < 0.7: + self.alu() + else: + self.ldst(wb=False) + self.out.append(f"9{n}:") def fold(self): for n in self.r.sample(POOL, 6): @@ -311,6 +395,9 @@ def program(self, n): self.emit(f"add x{b}, x0, x25") for n_ in POOL: self.movimm(n_, self.imm64()) + for n_ in VPOOL: + self.emit(f"fmov d{n_}, x{self.reg()}") + self.emit(f"mov v{n_}.d[1], x{self.reg()}") self.emit("msr nzcv, x1") i = 0 while i < n: @@ -332,6 +419,11 @@ def program(self, n): self.emit("ror x23, x23, #7") self.emit("mrs x22, nzcv") self.emit("eor x23, x23, x22") + for n_ in VPOOL: + self.emit(f"fmov x22, d{n_}") + self.emit("eor x23, x23, x22") + self.emit(f"mov x22, v{n_}.d[1]") + self.emit("eor x23, x23, x22, ror #17") # buffer base back from x26's final offset, for nic_hash_buffer self.movimm(25, BASES[26][0] + self.off[26]) self.emit("sub x0, x26, x25") diff --git a/include/tcg/tcg-sit.h b/include/tcg/tcg-sit.h index 1c6d437e37..fb39940a9c 100644 --- a/include/tcg/tcg-sit.h +++ b/include/tcg/tcg-sit.h @@ -47,6 +47,11 @@ typedef enum SitKind SIT_ST, SIT_LDP, SIT_STP, + /* Loads and stores of SIMD&FP registers (rt, rt2: V0-V31; size 0-4: B, H, S, D, Q). */ + SIT_LDV, + SIT_STV, + SIT_LDPV, + SIT_STPV, /* Conditional branches to label: B.cond (cond), CBZ/CBNZ, TBZ/TBNZ. These end the region. */ SIT_BCOND, SIT_CBZ, @@ -62,6 +67,7 @@ typedef enum SitAddrMode SIT_AM_PRE, /* [rn, #imm]! */ SIT_AM_POST, /* [rn], #imm */ SIT_AM_REG, /* [rn, rm, #shift] */ + SIT_AM_PC, /* literal: guest pc + imm (pcrel), or imm */ } SitAddrMode; /* SitInsn.ext for loads: what happens to the loaded value. */ @@ -110,6 +116,8 @@ typedef struct SitLayout int vf; /* uint32_t: V is bit 31 */ int nf; /* uint32_t: N is bit 31 */ int zf; /* uint32_t: Z is set if zero */ + int vregs; /* SIMD&FP register V0; low 128 bits little-endian */ + int vstride; /* bytes between V and V */ } SitLayout; #endif diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c index 65f21d3a93..2b1d6e5777 100644 --- a/target/arm/tcg/translate-a64.c +++ b/target/arm/tcg/translate-a64.c @@ -449,6 +449,8 @@ static const SitLayout a64_sit_layout = { .vf = offsetof(CPUARMState, VF), .nf = offsetof(CPUARMState, NF), .zf = offsetof(CPUARMState, ZF), + .vregs = offsetof(CPUARMState, vfp.zregs[0]), + .vstride = sizeof(ARMVectorReg), }; /* sit_copy() field uses: SIT_USE_R/W, and whether register 31 is SP (else XZR). */ @@ -544,16 +546,21 @@ static SitInsn* sit_ldst(DisasContext* s, SitKind kind, int rt, int rt2, int rn, int size, SitLoadExt ext, MemOp mop) { bool wb = mode == SIT_AM_PRE || mode == SIT_AM_POST; - bool pair = kind == SIT_LDP || kind == SIT_STP; + bool pair = kind == SIT_LDP || kind == SIT_STP || kind == SIT_LDPV || kind == SIT_STPV; SitInsn* si; if (!sit_admit(s, 0) || !tcg_native_int_ldst) { return NULL; } - if (wb && rn != 31 && (rn == rt || (pair && rn == rt2))) { return NULL; } - if (kind == SIT_LDP && rt == rt2) { return NULL; } + bool vec = kind >= SIT_LDV && kind <= SIT_STPV; + + if (vec && (dc_isar_feature(aa64_sve, s) || dc_isar_feature(aa64_sme, s))) { + return NULL; /* a SIMD&FP load would also have to clear the SVE part */ + } + if (wb && !vec && rn != 31 && (rn == rt || (pair && rn == rt2))) { return NULL; } + if ((kind == SIT_LDP || kind == SIT_LDPV) && rt == rt2) { return NULL; } si = sit_new(s, kind); - si->rt = sit_reg(rt, false); - si->rt2 = pair ? sit_reg(rt2, false) : SIT_REG_ZR; - si->rn = rn; + si->rt = vec ? rt : sit_reg(rt, false); + si->rt2 = !pair ? SIT_REG_ZR : vec ? rt2 : sit_reg(rt2, false); + si->rn = rn < 0 ? SIT_REG_ZR : rn; si->rm = SIT_REG_ZR; si->mode = mode; si->imm = imm; @@ -2987,6 +2994,24 @@ static bool sit_ldst_reg(DisasContext* s, SitKind kind, arg_ldst* a, SitLoadExt return true; } +/* LDR (literal), general-purpose or SIMD&FP: the address is pc_curr + @diff. */ +static bool sit_ldst_lit(DisasContext* s, SitKind kind, int rt, int64_t diff, int size, SitLoadExt ext, MemOp mop) +{ + SitInsn* si; + + if (s->pc_save == -1) { return false; } + si = sit_ldst(s, kind, rt, 0, -1, SIT_AM_PC, 0, size, ext, mop); + if (!si) { return false; } + if (tb_cflags(s->base.tb) & CF_PCREL) { + si->pcrel = true; + si->imm = (s->pc_curr - s->pc_save) + diff; + } + else { + si->imm = s->pc_curr + diff; + } + return true; +} + static bool trans_STXR(DisasContext* s, arg_stxr* a) { if (a->rn == 31) { gen_check_sp_alignment(s); } @@ -3075,10 +3100,17 @@ static bool trans_CAS(DisasContext* s, arg_CAS* a) static bool trans_LD_lit(DisasContext* s, arg_ldlit* a) { bool iss_sf = ldst_iss_sf(a->sz, a->sign, false); - TCGv_i64 tcg_rt = cpu_reg(s, a->rt); - TCGv_i64 clean_addr = tcg_temp_new_i64(); + TCGv_i64 tcg_rt; + TCGv_i64 clean_addr; MemOp memop = finalize_memop(s, a->sz + a->sign * MO_SIGN); + if (sit_ldst_lit(s, SIT_LD, a->rt, a->imm, a->sz, a->sign ? SIT_LD_SEXT : SIT_LD_ZEXT, memop)) { + sit_syndrome(s, memop, true, a->rt, iss_sf); + return true; + } + tcg_rt = cpu_reg(s, a->rt); + clean_addr = tcg_temp_new_i64(); + gen_pc_plus_diff(s, clean_addr, a->imm); do_gpr_ld(s, tcg_rt, clean_addr, memop, false, true, a->rt, iss_sf, false); return true; @@ -3092,6 +3124,7 @@ static bool trans_LD_lit_v(DisasContext* s, arg_ldlit* a) if (!fp_access_check(s)) { return true; } memop = finalize_memop_asimd(s, a->sz); + if (sit_ldst_lit(s, SIT_LDV, a->rt, a->imm, a->sz, SIT_LD_ZEXT, memop)) { return true; } clean_addr = tcg_temp_new_i64(); gen_pc_plus_diff(s, clean_addr, a->imm); do_fp_ld(s, a->rt, clean_addr, memop); @@ -3234,6 +3267,9 @@ static bool trans_STP_v(DisasContext* s, arg_ldstpair* a) /* LSE2 does not merge FP pairs; leave these as separate operations. */ mop = finalize_memop_asimd(s, a->sz); + if (sit_ldst(s, SIT_STPV, a->rt, a->rt2, a->rn, sit_mode(a->p, a->w), offset, a->sz, SIT_LD_ZEXT, mop)) { + return true; + } op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, true, mop); do_fp_st(s, a->rt, clean_addr, mop); tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz); @@ -3252,6 +3288,9 @@ static bool trans_LDP_v(DisasContext* s, arg_ldstpair* a) /* LSE2 does not merge FP pairs; leave these as separate operations. */ mop = finalize_memop_asimd(s, a->sz); + if (sit_ldst(s, SIT_LDPV, a->rt, a->rt2, a->rn, sit_mode(a->p, a->w), offset, a->sz, SIT_LD_ZEXT, mop)) { + return true; + } op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, false, mop); do_fp_ld(s, a->rt, clean_addr, mop); tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz); @@ -3380,6 +3419,9 @@ static bool trans_STR_v_i(DisasContext* s, arg_ldst_imm* a) if (!fp_access_check(s)) { return true; } mop = finalize_memop_asimd(s, a->sz); + if (sit_ldst(s, SIT_STV, a->rt, 0, a->rn, sit_mode(a->p, a->w), a->imm, a->sz, SIT_LD_ZEXT, mop)) { + return true; + } op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop); do_fp_st(s, a->rt, clean_addr, mop); op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); @@ -3393,6 +3435,9 @@ static bool trans_LDR_v_i(DisasContext* s, arg_ldst_imm* a) if (!fp_access_check(s)) { return true; } mop = finalize_memop_asimd(s, a->sz); + if (sit_ldst(s, SIT_LDV, a->rt, 0, a->rn, sit_mode(a->p, a->w), a->imm, a->sz, SIT_LD_ZEXT, mop)) { + return true; + } op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop); do_fp_ld(s, a->rt, clean_addr, mop); op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); @@ -3462,6 +3507,7 @@ static bool trans_LDR_v(DisasContext* s, arg_ldst* a) if (!fp_access_check(s)) { return true; } memop = finalize_memop_asimd(s, a->sz); + if (sit_ldst_reg(s, SIT_LDV, a, SIT_LD_ZEXT, memop)) { return true; } op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, false, memop); do_fp_ld(s, a->rt, clean_addr, memop); return true; @@ -3477,6 +3523,7 @@ static bool trans_STR_v(DisasContext* s, arg_ldst* a) if (!fp_access_check(s)) { return true; } memop = finalize_memop_asimd(s, a->sz); + if (sit_ldst_reg(s, SIT_STV, a, SIT_LD_ZEXT, memop)) { return true; } op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, true, memop); do_fp_st(s, a->rt, clean_addr, memop); return true; diff --git a/tcg/aarch64/tcg-target-sit.c.inc b/tcg/aarch64/tcg-target-sit.c.inc index ff74a144bc..fb06524883 100644 --- a/tcg/aarch64/tcg-target-sit.c.inc +++ b/tcg/aarch64/tcg-target-sit.c.inc @@ -323,16 +323,17 @@ static void sit_addpc(TCGContext* s, const SitInsn* si) /* * The softmmu TLB fast path for the guest address in @addr, as - * prepare_host_addr() but flag-free: leaves the host address addend in - * SIT_S1 and returns the CBNZ to patch with the slow path. + * prepare_host_addr() but flag-free, for an access of @span bytes with the + * atomicity and alignment of @oi: leaves the host address addend in SIT_S1 + * and returns the CBNZ to patch with the slow path. */ -static tcg_insn_unit* sit_tlb(TCGContext* s, TCGReg addr, MemOpIdx oi, bool is_ld) +static tcg_insn_unit* sit_tlb(TCGContext* s, TCGReg addr, MemOpIdx oi, bool is_ld, unsigned span) { MemOp opc = get_memop(oi); MemOp s_bits = opc & MO_SIZE; TCGAtomAlign aa = atom_and_align_for_opc(s, opc, have_lse2 ? MO_ATOM_WITHIN16 : MO_ATOM_IFALIGN, s_bits == MO_128); unsigned a_mask = (1 << aa.align) - 1; - unsigned s_mask = (1u << s_bits) - 1; + unsigned s_mask = span - 1; int fast_ofs = tlb_mask_table_ofs(s, get_mmuidx(oi)); uint64_t compare_mask = (uint64_t)TARGET_PAGE_MASK | a_mask; tcg_insn_unit* label_ptr; @@ -372,20 +373,51 @@ static AArch64Insn sit_ld_insn(const SitInsn* si) } } +static inline bool sit_is_vec(const SitInsn* si) { return si->kind >= SIT_LDV && si->kind <= SIT_STPV; } + +static inline bool sit_is_load(const SitInsn* si) +{ return si->kind == SIT_LD || si->kind == SIT_LDP || si->kind == SIT_LDV || si->kind == SIT_LDPV; } + +static inline bool sit_is_pair(const SitInsn* si) +{ return si->kind == SIT_LDP || si->kind == SIT_STP || si->kind == SIT_LDPV || si->kind == SIT_STPV; } + +/* The CPU state offset of guest vector register @v (its low 128 bits; little-endian). */ +static inline int sit_vofs(TCGContext* s, int v) { return s->sit->lay->vregs + s->sit->lay->vstride * v; } + +/* LDR/STR of 1 << @size bytes of a vector register (C3.3.12 form; size 4 is Q). */ +static AArch64Insn sit_vldst_insn(unsigned size, bool is_ld) +{ + unsigned opc = size == 4 ? (is_ld ? 3 : 2) : (is_ld ? 1 : 0); + + return 0x3c000000u | (size & 3) << 30 | opc << 22; +} + +/* Host vector scratch for vector loads and stores (TCG holds nothing in a region). */ +#define SIT_V0 TCG_REG_V31 +#define SIT_V1 TCG_REG_V30 + static void sit_ldst(TCGContext* s, const SitInsn* si) { static const AArch64Insn st[4] = {I3312_STRB, I3312_STRH, I3312_STRW, I3312_STRX}; SitMap* m = &s->sit->m; - bool is_ld = si->kind == SIT_LD || si->kind == SIT_LDP; - bool pair = si->kind == SIT_LDP || si->kind == SIT_STP; - uint32_t keep = sit_bit(si->rn) | sit_bit(si->rt) | (pair ? sit_bit(si->rt2) : 0); - TCGReg hn, hm = TCG_REG_XZR, ht, ht2 = TCG_REG_XZR, addr; + bool vec = sit_is_vec(si); + bool is_ld = sit_is_load(si); + bool pair = sit_is_pair(si); + unsigned span = (pair ? 2 : 1) << si->size; + uint32_t keep = 0; + TCGReg hn = TCG_REG_XZR, hm = TCG_REG_XZR, ht, ht2 = TCG_REG_XZR, addr; SitSlow* sp; + if (si->mode != SIT_AM_PC) { keep |= sit_bit(si->rn); } if (si->mode == SIT_AM_REG) { keep |= sit_bit(si->rm); } - hn = sit_use(s, si->rn, keep); + if (!vec) { keep |= sit_bit(si->rt) | (pair ? sit_bit(si->rt2) : 0); } + if (si->mode != SIT_AM_PC) { hn = sit_use(s, si->rn, keep); } if (si->mode == SIT_AM_REG) { hm = sit_use(s, si->rm, keep); } - if (is_ld) { + if (vec) { + ht = SIT_V0; + ht2 = SIT_V1; + } + else if (is_ld) { ht = sit_def(s, si->rt, keep); if (pair) { ht2 = sit_def(s, si->rt2, keep); } } @@ -417,10 +449,25 @@ static void sit_ldst(TCGContext* s, const SitInsn* si) addr = SIT_S3; tcg_out32(s, 0x8b200000u | hm << 16 | si->ext_opt << 13 | si->ext_shift << 10 | hn << 5 | addr); break; + case SIT_AM_PC: + addr = SIT_S3; + if (si->pcrel) { + tcg_out_ldst(s, I3312_LDRX, addr, TCG_AREG0, s->sit->lay->pc, 3); + sit_addi(s, addr, addr, si->imm); + } + else { + tcg_out_movi(s, TCG_TYPE_I64, addr, si->imm); + } + break; default: g_assert_not_reached(); } + if (vec && !is_ld) { + /* the data, from the CPU state (the TLB check below leaves vector registers alone) */ + tcg_out_ldst(s, I3312_LDRVQ, SIT_V0, TCG_AREG0, sit_vofs(s, si->rt), 4); + if (pair) { tcg_out_ldst(s, I3312_LDRVQ, SIT_V1, TCG_AREG0, sit_vofs(s, si->rt2), 4); } + } sp->addr = addr; - sp->label_ptr = sit_tlb(s, addr, si->oi, is_ld); + sp->label_ptr = sit_tlb(s, addr, si->oi, is_ld, span); switch (si->kind) { case SIT_LD: tcg_out_ldst_r(s, sit_ld_insn(si), ht, SIT_S1, TCG_TYPE_I64, addr); break; @@ -434,8 +481,28 @@ static void sit_ldst(TCGContext* s, const SitInsn* si) tcg_out32(s, opc | (is_ld ? 1u << 22 : 0) | ht2 << 10 | SIT_S1 << 5 | ht); break; } + case SIT_LDV: + case SIT_STV: tcg_out_ldst_r(s, sit_vldst_insn(si->size, is_ld), ht, SIT_S1, TCG_TYPE_I64, addr); break; + case SIT_LDPV: + case SIT_STPV: { + /* LDP/STP St/Dt/Qt, [x17] (signed offset 0): opc 0 S, 1 D, 2 Q */ + uint32_t opc = si->size - 2; + + tcg_out_insn(s, 3502, ADD, TCG_TYPE_I64, SIT_S1, SIT_S1, addr); + tcg_out32(s, 0x2d000000u | opc << 30 | (is_ld ? 1u << 22 : 0) | (ht2 & 31) << 10 | SIT_S1 << 5 | (ht & 31)); + break; + } default: g_assert_not_reached(); } + if (vec && is_ld) { + /* + * To the CPU state: all 128 bits, as the host load zeroed what is above the + * access, which is what a guest load into a SIMD&FP register does (no SVE). + * The slow path writes the CPU state itself and rejoins after this. + */ + tcg_out_ldst(s, I3312_STRVQ, SIT_V0, TCG_AREG0, sit_vofs(s, si->rt), 4); + if (pair) { tcg_out_ldst(s, I3312_STRVQ, SIT_V1, TCG_AREG0, sit_vofs(s, si->rt2), 4); } + } sp->raddr = tcg_splitwx_to_rx(s->code_ptr); QSIMPLEQ_INSERT_TAIL(&s->sit->slow, sp, next); @@ -443,7 +510,7 @@ static void sit_ldst(TCGContext* s, const SitInsn* si) sit_addi(s, hn, hn, si->imm); sit_dirty(s, si->rn); } - if (is_ld) { + if (is_ld && !vec) { sit_dirty(s, si->rt); if (pair) { sit_dirty(s, si->rt2); } } @@ -486,7 +553,11 @@ static void tcg_out_sit(TCGContext* s, const SitInsn* si, TCGLabel* l) case SIT_LD : case SIT_ST : case SIT_LDP : - case SIT_STP : sit_ldst(s, si); break; + case SIT_STP : + case SIT_LDV : + case SIT_STV : + case SIT_LDPV : + case SIT_STPV : sit_ldst(s, si); break; case SIT_BCOND: case SIT_CBZ : case SIT_TBZ : sit_branch(s, si, l); break; @@ -494,25 +565,77 @@ static void tcg_out_sit(TCGContext* s, const SitInsn* si, TCGLabel* l) } } +/* x1 = guest address, x2... = data, then oi and the return address; call the TCG helper. */ +static void sit_slow_call(TCGContext* s, const SitSlow* sp, MemOp mop, bool is_ld, TCGReg data_hi_used) +{ + TCGReg next = is_ld ? TCG_REG_X2 : data_hi_used; + + tcg_out_movi(s, TCG_TYPE_I32, next, sp->si->oi); + tcg_out_movi(s, TCG_TYPE_PTR, next + 1, (uintptr_t)sp->raddr); + tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0); + tcg_out_call_int(s, is_ld ? qemu_ld_helpers[mop & MO_SSIZE] : qemu_st_helpers[mop & MO_SIZE]); +} + +/* Guest vector register @v (low 128 bits) = x0 (and x1 if @wide, else zero above 64 bits). */ +static void sit_slow_vset(TCGContext* s, int v, bool wide) +{ + int o = sit_vofs(s, v); + + tcg_out_ldst(s, I3312_STRX, TCG_REG_X0, TCG_AREG0, o, 3); + tcg_out_ldst(s, I3312_STRX, wide ? TCG_REG_X1 : TCG_REG_XZR, TCG_AREG0, o + 8, 3); +} + +/* + * A vector load or store in the slow path: one helper call per register. + * The address of the second register is kept in the outgoing-argument area of + * the TCG frame across the first call. + */ +static void sit_slow_vec(TCGContext* s, const SitSlow* sp) +{ + const SitInsn* si = sp->si; + MemOp mop = get_memop(si->oi); + bool is_ld = sit_is_load(si); + bool wide = si->size == 4; + int n = sit_is_pair(si) ? 2 : 1; + + tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_X1, sp->addr); + if (n == 2) { tcg_out_ldst(s, I3312_STRX, TCG_REG_X1, TCG_REG_SP, 0, 3); } + for (int i = 0; i < n; i++) { + int v = i ? si->rt2 : si->rt; + + if (i) { + tcg_out_ldst(s, I3312_LDRX, TCG_REG_X1, TCG_REG_SP, 0, 3); + tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_X1, TCG_REG_X1, 1 << si->size); + } + if (is_ld) { + sit_slow_call(s, sp, mop, true, TCG_REG_X2); + sit_slow_vset(s, v, wide); + } + else { + tcg_out_ldst(s, I3312_LDRX, TCG_REG_X2, TCG_AREG0, sit_vofs(s, v), 3); + if (wide) { tcg_out_ldst(s, I3312_LDRX, TCG_REG_X3, TCG_AREG0, sit_vofs(s, v) + 8, 3); } + sit_slow_call(s, sp, mop, false, wide ? TCG_REG_X4 : TCG_REG_X3); + } + } +} + /* The out-of-line path of a SIT load or store: the CPU state is made exact, then the TCG helper. */ static bool sit_slow_out(TCGContext* s, SitSlow* sp) { - const SitInsn* si = sp->si; - MemOp mop = get_memop(si->oi); - bool is_ld = si->kind == SIT_LD || si->kind == SIT_LDP; - uint32_t reload; + const SitInsn* si = sp->si; + MemOp mop = get_memop(si->oi); + bool is_ld = sit_is_load(si); + uint32_t reload; if (!reloc_pc19(sp->label_ptr, tcg_splitwx_to_rx(s->code_ptr))) { return false; } if (!tcg_native_int_fault) { sit_writeback(s, &sp->map); } - /* env, addr, [data], oi, retaddr */ - if (is_ld) { + if (sit_is_vec(si)) { sit_slow_vec(s, sp); } + else if (is_ld) { + /* env, addr, oi, retaddr */ tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_X1, sp->addr); - tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, si->oi); - tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_X3, (uintptr_t)sp->raddr); - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0); - tcg_out_call_int(s, qemu_ld_helpers[mop & MO_SSIZE]); + sit_slow_call(s, sp, mop, true, TCG_REG_X2); /* Results to the CPU state; the reload below brings them into the data registers. */ if (si->kind == SIT_LD) { @@ -538,7 +661,7 @@ static bool sit_slow_out(TCGContext* s, SitSlow* sp) } } else { - TCGReg data = TCG_REG_X2; + TCGReg next = TCG_REG_X3; sit_mov(s, SIT_S0, sp->rt); if (si->kind == SIT_STP) { @@ -551,16 +674,13 @@ static bool sit_slow_out(TCGContext* s, SitSlow* sp) } sit_mov(s, SIT_S1, sp->addr); tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_X1, SIT_S1); - tcg_out_mov(s, TCG_TYPE_I64, data, SIT_S0); + tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_X2, SIT_S0); if (si->kind == SIT_STP && si->size == 3) { /* Int128 in x2:x3 (an even pair on Linux, the next two registers on Darwin: the same here) */ tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_X3, SIT_S2); - data = TCG_REG_X3; + next = TCG_REG_X4; } - tcg_out_movi(s, TCG_TYPE_I32, data + 1, si->oi); - tcg_out_movi(s, TCG_TYPE_PTR, data + 2, (uintptr_t)sp->raddr); - tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0); - tcg_out_call_int(s, qemu_st_helpers[mop & MO_SIZE]); + sit_slow_call(s, sp, mop, false, next); } /* Reload what the call clobbered, and the loaded registers. */ @@ -568,7 +688,7 @@ static bool sit_slow_out(TCGContext* s, SitSlow* sp) for (int g = 0; g < SIT_NUM_REGS; g++) { if (sp->map.host[g] >= 0 && (SIT_CALL_CLOBBERED & (1u << sp->map.host[g]))) { reload |= 1u << g; } } - if (is_ld) { + if (is_ld && !sit_is_vec(si)) { reload |= sit_bit(si->rt); if (si->kind == SIT_LDP) { reload |= sit_bit(si->rt2); } }