Skip to content

native-int: SIMD&FP loads/stores and literal loads stay inside regions #61

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dylanneve1 merged 1 commit into from
Oct 22, 2026
Merged

native-int: SIMD&FP loads/stores and literal loads stay inside regions #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/NATIVE-INT.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,9 @@ EL0, AArch64, little-endian data, no single step, no BTI, no MTE:
|---|---|
| data processing | ADD/SUB(S) immediate, shifted and extended register; AND/ORR/EOR/ANDS (immediate) and the shifted-register logical forms; MOVZ/MOVN/MOVK; SBFM/UBFM/BFM; EXTR; MADD/MSUB/SMADDL/SMSUBL/UMADDL/UMSUBL/SMULH/UMULH; UDIV/SDIV; LSLV/LSRV/ASRV/RORV; RBIT/REV16/REV32/REV/CLZ/CLS; ADC/SBC(S); CCMP/CCMN; CSEL/CSINC/CSINV/CSNEG |
| PC-relative | ADR, ADRP |
| loads/stores | LDR/STR of general-purpose registers: unsigned and unscaled offsets, pre/post-index, register offset, sign-extending loads; LDP/STP/LDPSW |
| loads/stores | LDR/STR of general-purpose registers: unsigned and unscaled offsets, pre/post-index, register offset, sign-extending loads; LDP/STP/LDPSW; LDR (literal) |
| SIMD&FP loads/stores | LDR/STR of B/H/S/D/Q registers (same addressing modes), LDP/STP of S/D/Q registers, LDR (literal); no SVE/SME. The data goes through the CPU state (vector registers are not mapped) |
| system registers | MRS of plain 64-bit fields with no access check (TPIDR_EL0, TPIDRRO_EL0) |
| branches | B.cond (when the guest flags are already in the host flags), CBZ/CBNZ, TBZ/TBNZ (inside a region) |

Everything else stays on the existing paths, in particular system registers,
Expand Down
32 changes: 32 additions & 0 deletions hw/arm/tests/native_int_check.S
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
* 4 x18, x29 and x30 as ordinary operands and load destinations
* 5 NZCV set before a page-crossing (slow-path) load and store and
* consumed after them
* 8 SIMD&FP loads in a region: a Q load and a pair that cross a 16 KiB
* page (slow path), and a Q load that faults: the destination stays
* unchanged and the registers written before it are exact
* 7 MRS TPIDR_EL0 / TPIDRRO_EL0 (plain field reads) in the middle of
* ALU code, after MSR TPIDR_EL0 at EL0 and TPIDRRO_EL0 set at EL1
* 6 a store over code later in the same block (the store's slow
Expand Down Expand Up @@ -70,6 +73,9 @@
.text
.globl _start
_start:
mrs x0, cpacr_el1
orr x0, x0, #(3 << 20) /* FPEN: SIMD&FP at EL0 */
msr cpacr_el1, x0
adr x0, vectors
msr vbar_el1, x0
ldr x0, =EL1_SP
Expand Down Expand Up @@ -307,6 +313,32 @@ fault2:
CHK x15, 0x1111 - 0x2222, 6
CHK x17, 1, 6

/* 8: SIMD&FP loads and stores */
LI x0, BUF + 0x4000 - 8 /* 16 bytes across a 16 KiB page */
LI x1, 0x1111222233334444
LI x2, 0x5555666677778888
stp x1, x2, [x0]
ldr q0, [x0]
ldp d1, d2, [x0]
str q0, [x0, #-64]
ldp x3, x4, [x0, #-64]
fmov x5, d1
fmov x6, d2
CHK x3, 0x1111222233334444, 8
CHK x4, 0x5555666677778888, 8
CHK x5, 0x1111222233334444, 8
CHK x6, 0x5555666677778888, 8
movi v3.2d, #0xffffffffffffffff
LI x9, HOLE + 0x200
movz x10, #0x2a
add x11, x10, #1
fault8:
ldr q3, [x9]
CHKSAVE 11, 0x2b, 8
CHKSAVE 31, HOLE + 0x200, 8
fmov x12, d3
CHK x12, 0xffffffffffffffff, 8 /* destination unchanged */

/* 7: TLS register reads in the middle of a block */
LI x0, 0x0123456789abcdef
msr tpidr_el0, x0
Expand Down
102 changes: 97 additions & 5 deletions hw/arm/tests/native_int_gen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
import sys

POOL = list(range(0, 18)) + [19, 20, 21]
# SIMD&FP registers the program may use: V8-V15 are callee-saved on the host (the harness).
VPOOL = list(range(0, 8)) + list(range(16, 32))
VSZ = {0: "b", 1: "h", 2: "s", 3: "d", 4: "q"}
CONDS = ["eq", "ne", "cs", "cc", "mi", "pl", "vs", "vc", "hi", "ls", "ge", "lt", "gt", "le", "al"]
# buffer windows: base register, window start (offset in the 64 KiB buffer), initial offset in the window.
# x27 sits 64 bytes below a 16 KiB page boundary, so its accesses cross pages (the TLB slow path).
Expand Down Expand Up @@ -173,9 +176,86 @@ def window(self, b):
start = BASES[b][0]
return start, start + WINDOW

def vldst(self, wb=True):
"""SIMD&FP loads and stores (and literal loads), like ldst()."""
r = self.r
b = r.choice(list(BASES))
cur = self.off[b]
k = r.randrange(8)
v = lambda: r.choice(VPOOL) # noqa: E731
if k <= 2:
size = r.randrange(5)
op = r.choice(["ldr", "str"])
mode = r.choice(["uoff", "unscaled", "pre", "post"] if wb else ["uoff", "unscaled"])
reg = f"{VSZ[size]}{v()}"
if mode == "uoff":
off = r.randrange(0, 128) << size
if cur + off + (1 << size) > WINDOW:
off = 0
self.emit(f"{op} {reg}, [x{b}, #{off}]")
elif mode == "unscaled":
off = r.randrange(-256, 256)
if not (0 <= cur + off and cur + off + 16 <= WINDOW):
off = 0
self.emit(f"{op.replace('ldr', 'ldur').replace('str', 'stur')} {reg}, [x{b}, #{off}]")
else:
imm = r.randrange(-256, 256)
lo, hi = 64, WINDOW - 64
if not (lo <= cur + imm <= hi):
imm = -imm if lo <= cur - imm <= hi and -imm != 256 else 0
self.emit(f"{op} {reg}, [x{b}, #{imm}]!" if mode == "pre" else f"{op} {reg}, [x{b}], #{imm}")
self.off[b] = cur + imm
elif k <= 3:
size = r.randrange(5)
op = r.choice(["ldr", "str"])
self.emit(f"and x25, {self.x()}, #0xf8")
ext = r.choice(["lsl", "uxtw", "sxtw"])
idx = "x25" if ext == "lsl" else "w25"
sh = r.choice([0, size]) if size else 0
if ext == "lsl" and not sh:
self.emit(f"{op} {VSZ[size]}{v()}, [x{b}, {idx}]")
else:
self.emit(f"{op} {VSZ[size]}{v()}, [x{b}, {idx}, {ext} #{sh}]")
elif k <= 5:
size = r.randrange(2, 5)
op = r.choice(["ldp", "stp"])
a_, c_ = v(), v()
while op == "ldp" and c_ == a_:
c_ = v()
scale = 1 << size
mode = r.choice(["off", "pre", "post"] if wb else ["off"])
imm = r.randrange(-64, 64) * scale
lo, hi = 128, WINDOW - 128
if not (lo <= cur + imm <= hi):
imm = -imm if lo <= cur - imm <= hi and -imm != 64 * scale else 0
regs = f"{VSZ[size]}{a_}, {VSZ[size]}{c_}"
if mode == "off":
self.emit(f"{op} {regs}, [x{b}, #{imm}]")
elif mode == "pre":
self.emit(f"{op} {regs}, [x{b}, #{imm}]!")
self.off[b] = cur + imm
else:
self.emit(f"{op} {regs}, [x{b}], #{imm}")
self.off[b] = cur + imm
else:
# literal loads from a pool jumped over
self.label += 1
n = self.label
if r.random() < 0.5:
self.emit(f"ldr {VSZ[r.randrange(2, 5)]}{v()}, 8{n}f")
else:
op, dst = r.choice([("ldr", self.x()), ("ldr", self.w()), ("ldrsw", self.x())])
self.emit(f"{op} {dst}, 8{n}f")
self.emit(f"b 9{n}f")
self.out.append(f"8{n}:")
self.emit(f".quad {r.getrandbits(64):#x}, {r.getrandbits(64):#x}")
self.out.append(f"9{n}:")

def ldst(self, wb=True):
"""wb=False: no pre/post-index forms (inside a branch shadow the tracked base offsets must not move)."""
r = self.r
if r.random() < 0.3:
return self.vldst(wb)
b = r.choice(list(BASES))
cur = self.off[b]
k = r.randrange(10)
Expand Down Expand Up @@ -286,16 +366,20 @@ def other(self):
def branch(self):
r = self.r
self.label += 1
n = self.label
k = r.randrange(3)
if k == 0:
self.emit(f"b.{r.choice(CONDS[:-1])} 9{self.label}f")
self.emit(f"b.{r.choice(CONDS[:-1])} 9{n}f")
elif k == 1:
self.emit(f"{r.choice(['cbz', 'cbnz'])} {self.rr(r.random() < 0.5)}, 9{self.label}f")
self.emit(f"{r.choice(['cbz', 'cbnz'])} {self.rr(r.random() < 0.5)}, 9{n}f")
else:
self.emit(f"{r.choice(['tbz', 'tbnz'])} {self.x()}, #{r.randrange(64)}, 9{self.label}f")
self.emit(f"{r.choice(['tbz', 'tbnz'])} {self.x()}, #{r.randrange(64)}, 9{n}f")
for _ in range(r.randint(0, 3)):
self.alu() if r.random() < 0.7 else self.ldst(wb=False)
self.out.append(f"9{self.label}:")
if r.random() < 0.7:
self.alu()
else:
self.ldst(wb=False)
self.out.append(f"9{n}:")

def fold(self):
for n in self.r.sample(POOL, 6):
Expand All @@ -311,6 +395,9 @@ def program(self, n):
self.emit(f"add x{b}, x0, x25")
for n_ in POOL:
self.movimm(n_, self.imm64())
for n_ in VPOOL:
self.emit(f"fmov d{n_}, x{self.reg()}")
self.emit(f"mov v{n_}.d[1], x{self.reg()}")
self.emit("msr nzcv, x1")
i = 0
while i < n:
Expand All @@ -332,6 +419,11 @@ def program(self, n):
self.emit("ror x23, x23, #7")
self.emit("mrs x22, nzcv")
self.emit("eor x23, x23, x22")
for n_ in VPOOL:
self.emit(f"fmov x22, d{n_}")
self.emit("eor x23, x23, x22")
self.emit(f"mov x22, v{n_}.d[1]")
self.emit("eor x23, x23, x22, ror #17")
# buffer base back from x26's final offset, for nic_hash_buffer
self.movimm(25, BASES[26][0] + self.off[26])
self.emit("sub x0, x26, x25")
Expand Down
8 changes: 8 additions & 0 deletions include/tcg/tcg-sit.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ typedef enum SitKind
SIT_ST,
SIT_LDP,
SIT_STP,
/* Loads and stores of SIMD&FP registers (rt, rt2: V0-V31; size 0-4: B, H, S, D, Q). */
SIT_LDV,
SIT_STV,
SIT_LDPV,
SIT_STPV,
/* Conditional branches to label: B.cond (cond), CBZ/CBNZ, TBZ/TBNZ. These end the region. */
SIT_BCOND,
SIT_CBZ,
Expand All @@ -62,6 +67,7 @@ typedef enum SitAddrMode
SIT_AM_PRE, /* [rn, #imm]! */
SIT_AM_POST, /* [rn], #imm */
SIT_AM_REG, /* [rn, rm, <ext> #shift] */
SIT_AM_PC, /* literal: guest pc + imm (pcrel), or imm */
} SitAddrMode;

/* SitInsn.ext for loads: what happens to the loaded value. */
Expand Down Expand Up @@ -110,6 +116,8 @@ typedef struct SitLayout
int vf; /* uint32_t: V is bit 31 */
int nf; /* uint32_t: N is bit 31 */
int zf; /* uint32_t: Z is set if zero */
int vregs; /* SIMD&FP register V0; low 128 bits little-endian */
int vstride; /* bytes between V<n> and V<n+1> */
} SitLayout;

#endif
63 changes: 55 additions & 8 deletions target/arm/tcg/translate-a64.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,8 @@ static const SitLayout a64_sit_layout = {
.vf = offsetof(CPUARMState, VF),
.nf = offsetof(CPUARMState, NF),
.zf = offsetof(CPUARMState, ZF),
.vregs = offsetof(CPUARMState, vfp.zregs[0]),
.vstride = sizeof(ARMVectorReg),
};

/* sit_copy() field uses: SIT_USE_R/W, and whether register 31 is SP (else XZR). */
Expand Down Expand Up @@ -544,16 +546,21 @@ static SitInsn* sit_ldst(DisasContext* s, SitKind kind, int rt, int rt2, int rn,
int size, SitLoadExt ext, MemOp mop)
{
bool wb = mode == SIT_AM_PRE || mode == SIT_AM_POST;
bool pair = kind == SIT_LDP || kind == SIT_STP;
bool pair = kind == SIT_LDP || kind == SIT_STP || kind == SIT_LDPV || kind == SIT_STPV;
SitInsn* si;

if (!sit_admit(s, 0) || !tcg_native_int_ldst) { return NULL; }
if (wb && rn != 31 && (rn == rt || (pair && rn == rt2))) { return NULL; }
if (kind == SIT_LDP && rt == rt2) { return NULL; }
bool vec = kind >= SIT_LDV && kind <= SIT_STPV;

if (vec && (dc_isar_feature(aa64_sve, s) || dc_isar_feature(aa64_sme, s))) {
return NULL; /* a SIMD&FP load would also have to clear the SVE part */
}
if (wb && !vec && rn != 31 && (rn == rt || (pair && rn == rt2))) { return NULL; }
if ((kind == SIT_LDP || kind == SIT_LDPV) && rt == rt2) { return NULL; }
si = sit_new(s, kind);
si->rt = sit_reg(rt, false);
si->rt2 = pair ? sit_reg(rt2, false) : SIT_REG_ZR;
si->rn = rn;
si->rt = vec ? rt : sit_reg(rt, false);
si->rt2 = !pair ? SIT_REG_ZR : vec ? rt2 : sit_reg(rt2, false);
si->rn = rn < 0 ? SIT_REG_ZR : rn;
si->rm = SIT_REG_ZR;
si->mode = mode;
si->imm = imm;
Expand Down Expand Up @@ -2987,6 +2994,24 @@ static bool sit_ldst_reg(DisasContext* s, SitKind kind, arg_ldst* a, SitLoadExt
return true;
}

/* LDR (literal), general-purpose or SIMD&FP: the address is pc_curr + @diff. */
static bool sit_ldst_lit(DisasContext* s, SitKind kind, int rt, int64_t diff, int size, SitLoadExt ext, MemOp mop)
{
SitInsn* si;

if (s->pc_save == -1) { return false; }
si = sit_ldst(s, kind, rt, 0, -1, SIT_AM_PC, 0, size, ext, mop);
if (!si) { return false; }
if (tb_cflags(s->base.tb) & CF_PCREL) {
si->pcrel = true;
si->imm = (s->pc_curr - s->pc_save) + diff;
}
else {
si->imm = s->pc_curr + diff;
}
return true;
}

static bool trans_STXR(DisasContext* s, arg_stxr* a)
{
if (a->rn == 31) { gen_check_sp_alignment(s); }
Expand Down Expand Up @@ -3075,10 +3100,17 @@ static bool trans_CAS(DisasContext* s, arg_CAS* a)
static bool trans_LD_lit(DisasContext* s, arg_ldlit* a)
{
bool iss_sf = ldst_iss_sf(a->sz, a->sign, false);
TCGv_i64 tcg_rt = cpu_reg(s, a->rt);
TCGv_i64 clean_addr = tcg_temp_new_i64();
TCGv_i64 tcg_rt;
TCGv_i64 clean_addr;
MemOp memop = finalize_memop(s, a->sz + a->sign * MO_SIGN);

if (sit_ldst_lit(s, SIT_LD, a->rt, a->imm, a->sz, a->sign ? SIT_LD_SEXT : SIT_LD_ZEXT, memop)) {
sit_syndrome(s, memop, true, a->rt, iss_sf);
return true;
}
tcg_rt = cpu_reg(s, a->rt);
clean_addr = tcg_temp_new_i64();

gen_pc_plus_diff(s, clean_addr, a->imm);
do_gpr_ld(s, tcg_rt, clean_addr, memop, false, true, a->rt, iss_sf, false);
return true;
Expand All @@ -3092,6 +3124,7 @@ static bool trans_LD_lit_v(DisasContext* s, arg_ldlit* a)

if (!fp_access_check(s)) { return true; }
memop = finalize_memop_asimd(s, a->sz);
if (sit_ldst_lit(s, SIT_LDV, a->rt, a->imm, a->sz, SIT_LD_ZEXT, memop)) { return true; }
clean_addr = tcg_temp_new_i64();
gen_pc_plus_diff(s, clean_addr, a->imm);
do_fp_ld(s, a->rt, clean_addr, memop);
Expand Down Expand Up @@ -3234,6 +3267,9 @@ static bool trans_STP_v(DisasContext* s, arg_ldstpair* a)

/* LSE2 does not merge FP pairs; leave these as separate operations. */
mop = finalize_memop_asimd(s, a->sz);
if (sit_ldst(s, SIT_STPV, a->rt, a->rt2, a->rn, sit_mode(a->p, a->w), offset, a->sz, SIT_LD_ZEXT, mop)) {
return true;
}
op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, true, mop);
do_fp_st(s, a->rt, clean_addr, mop);
tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz);
Expand All @@ -3252,6 +3288,9 @@ static bool trans_LDP_v(DisasContext* s, arg_ldstpair* a)

/* LSE2 does not merge FP pairs; leave these as separate operations. */
mop = finalize_memop_asimd(s, a->sz);
if (sit_ldst(s, SIT_LDPV, a->rt, a->rt2, a->rn, sit_mode(a->p, a->w), offset, a->sz, SIT_LD_ZEXT, mop)) {
return true;
}
op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, false, mop);
do_fp_ld(s, a->rt, clean_addr, mop);
tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz);
Expand Down Expand Up @@ -3380,6 +3419,9 @@ static bool trans_STR_v_i(DisasContext* s, arg_ldst_imm* a)

if (!fp_access_check(s)) { return true; }
mop = finalize_memop_asimd(s, a->sz);
if (sit_ldst(s, SIT_STV, a->rt, 0, a->rn, sit_mode(a->p, a->w), a->imm, a->sz, SIT_LD_ZEXT, mop)) {
return true;
}
op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop);
do_fp_st(s, a->rt, clean_addr, mop);
op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
Expand All @@ -3393,6 +3435,9 @@ static bool trans_LDR_v_i(DisasContext* s, arg_ldst_imm* a)

if (!fp_access_check(s)) { return true; }
mop = finalize_memop_asimd(s, a->sz);
if (sit_ldst(s, SIT_LDV, a->rt, 0, a->rn, sit_mode(a->p, a->w), a->imm, a->sz, SIT_LD_ZEXT, mop)) {
return true;
}
op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop);
do_fp_ld(s, a->rt, clean_addr, mop);
op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
Expand Down Expand Up @@ -3462,6 +3507,7 @@ static bool trans_LDR_v(DisasContext* s, arg_ldst* a)
if (!fp_access_check(s)) { return true; }

memop = finalize_memop_asimd(s, a->sz);
if (sit_ldst_reg(s, SIT_LDV, a, SIT_LD_ZEXT, memop)) { return true; }
op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, false, memop);
do_fp_ld(s, a->rt, clean_addr, memop);
return true;
Expand All @@ -3477,6 +3523,7 @@ static bool trans_STR_v(DisasContext* s, arg_ldst* a)
if (!fp_access_check(s)) { return true; }

memop = finalize_memop_asimd(s, a->sz);
if (sit_ldst_reg(s, SIT_STV, a, SIT_LD_ZEXT, memop)) { return true; }
op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, true, memop);
do_fp_st(s, a->rt, clean_addr, memop);
return true;
Expand Down
Loading
Loading