diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c index 25c1ca653f..71072ccf1f 100644 --- a/accel/tcg/cpu-exec.c +++ b/accel/tcg/cpu-exec.c @@ -313,6 +313,19 @@ const void* HELPER(lookup_tb_ptr)(CPUArchState* env) return lookup_tb_ptr(cpu, s.pc, s.cs_base, s.flags); } +/* + * The miss path of translator_lookup_and_goto_ptr()'s inline jump-cache + * probe: the generated code has already looked in the jump cache and knows + * the TB state (that of the TB it belongs to), so go straight to the slow + * part. + */ +const void* HELPER(lookup_tb_ptr_miss)(CPUArchState* env, uint64_t pc, uint64_t cs_base, uint32_t flags) +{ + CPUState* cpu = env_cpu(env); + + return tb_lookup_ptr_slow(cpu, pc, cs_base, flags, curr_cflags(cpu)); +} + /* Return the current PC from CPU, which may be cached in TB. */ static vaddr log_pc(CPUState* cpu, const TranslationBlock* tb) { diff --git a/accel/tcg/internal-common.h b/accel/tcg/internal-common.h index 30032035b7..623219adb2 100644 --- a/accel/tcg/internal-common.h +++ b/accel/tcg/internal-common.h @@ -48,6 +48,9 @@ void tcg_exec_unrealizefn(CPUState* cpu); * Current cflags for hashing/comparison. Inline: it runs on every TB lookup, * including each indirect branch (helper_lookup_tb_ptr). */ +/* -accel tcg,x-inline-lookup=on|off (default on): see translator_lookup_and_goto_ptr() */ +extern bool tcg_inline_lookup; + static inline uint32_t curr_cflags(CPUState* cpu) { uint32_t cflags = cpu->tcg_cflags; diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c index 02a9d90672..42e2f9cf34 100644 --- a/accel/tcg/tcg-all.c +++ b/accel/tcg/tcg-all.c @@ -89,6 +89,7 @@ bool tcg_native_fp_explicit; bool tcg_native_fp_fault; bool tcg_native_fp_selftest = true; bool tcg_native_fp_inline = true; +bool tcg_inline_lookup = true; bool tcg_native_int; bool tcg_native_int_explicit; bool tcg_native_int_ldst = true; @@ -296,6 +297,8 @@ static void tcg_set_native_fp_selftest(Object* obj, bool value, Error** errp) { static bool tcg_get_native_fp_inline(Object* obj, Error** errp) { return tcg_native_fp_inline; } static void tcg_set_native_fp_inline(Object* obj, bool value, Error** errp) { tcg_native_fp_inline = value; } +static bool tcg_get_inline_lookup(Object* obj, Error** errp) { return tcg_inline_lookup; } +static void tcg_set_inline_lookup(Object* obj, bool value, Error** errp) { tcg_inline_lookup = value; } static bool tcg_get_native_int(Object* obj, Error** errp) { return tcg_native_int; } static void tcg_set_native_int(Object* obj, bool value, Error** errp) { @@ -358,6 +361,10 @@ static void tcg_accel_class_init(ObjectClass* oc, const void* data) object_class_property_set_description(oc, "x-native-fp-inline", "Diagnostic: set off to keep native-fp in helpers (no inline host FP)"); + object_class_property_add_bool(oc, "x-inline-lookup", tcg_get_inline_lookup, tcg_set_inline_lookup); + object_class_property_set_description(oc, "x-inline-lookup", + "Diagnostic: set off to do every indirect-branch TB lookup in the helper"); + object_class_property_add_bool(oc, "native-int", tcg_get_native_int, tcg_set_native_int); object_class_property_set_description(oc, "native-int", "Execute guest EL0 integer code, loads/stores and branches as the same " diff --git a/accel/tcg/tcg-runtime.h b/accel/tcg/tcg-runtime.h index 197554ae2f..c94049df76 100644 --- a/accel/tcg/tcg-runtime.h +++ b/accel/tcg/tcg-runtime.h @@ -25,6 +25,7 @@ DEF_HELPER_FLAGS_1(ctpop_i32, TCG_CALL_NO_RWG_SE, i32, i32) DEF_HELPER_FLAGS_1(ctpop_i64, TCG_CALL_NO_RWG_SE, i64, i64) DEF_HELPER_FLAGS_1(lookup_tb_ptr, TCG_CALL_NO_WG_SE, cptr, env) +DEF_HELPER_FLAGS_4(lookup_tb_ptr_miss, TCG_CALL_NO_WG_SE, cptr, env, i64, i64, i32) DEF_HELPER_FLAGS_1(exit_atomic, TCG_CALL_NO_WG, noreturn, env) diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c index e6fe77d3da..00add25b36 100644 --- a/accel/tcg/translator.c +++ b/accel/tcg/translator.c @@ -18,6 +18,9 @@ #include "tcg/tcg-op-common.h" #include "internal-common.h" #include "tb-internal.h" +#include "tb-hash.h" +#include "hw/core/cpu.h" +#include "exec/helper-gen-common.h" bool translator_io_start(DisasContextBase* db) { @@ -58,6 +61,92 @@ static void gen_tb_end(const TranslationBlock* tb, uint32_t cflags, int num_insn } } +/* + * Look up the TB for the guest pc in @pc and jump to it, like + * tcg_gen_lookup_and_goto_ptr_via(@gen_lookup), but with the jump-cache hit + * done inline in the generated code: hash the pc, load the cache entry, + * check its pc and the TB's cs_base, flags and cflags against those of the + * TB being translated, check for breakpoints, and jump to its code. Anything + * else (a miss, another TB state, breakpoints) calls helper_lookup_tb_ptr_miss(), + * which skips the state computation and the jump-cache probe that @gen_lookup + * would repeat. + * + * Only valid when the CPU state at this exit is the state of the TB being + * translated (@db->tb flags and cs_base) and its cflags are the vCPU's + * current ones: the caller asserts the former; the latter is checked here. + * The jump-cache hit in C (lookup_tb_ptr()) costs a helper call and some 70 + * host instructions; this is about 25. -accel tcg,x-inline-lookup=off keeps + * the helper for every lookup. + */ +void translator_lookup_and_goto_ptr(DisasContextBase* db, TCGv_i64 pc, void (*gen_lookup)(TCGv_ptr, TCGv_env)) +{ + const TranslationBlock* tb = db->tb; + TCGLabel* slow; + TCGv_ptr jc, ent, tbp, code; + TCGv_i64 h, t, v; + int shift = TARGET_PAGE_BITS - TB_JMP_PAGE_BITS; + + if (!tcg_inline_lookup || (tb_cflags(tb) & CF_NO_GOTO_PTR) || tb_cflags(tb) != curr_cflags(tcg_ctx->cpu) || + qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) { + tcg_gen_lookup_and_goto_ptr_via(gen_lookup); + return; + } + + slow = gen_new_label(); + h = tcg_temp_new_i64(); + t = tcg_temp_new_i64(); + v = tcg_temp_new_i64(); + jc = tcg_temp_new_ptr(); + ent = tcg_temp_new_ptr(); + tbp = tcg_temp_new_ptr(); + code = tcg_temp_new_ptr(); + + /* h = tb_jmp_cache_hash_func(pc) */ + QEMU_BUILD_BUG_ON(sizeof(((CPUJumpCache*)0)->array[0]) != 16); + tcg_gen_shri_i64(t, pc, shift); + tcg_gen_xor_i64(t, t, pc); + tcg_gen_shri_i64(t, t, shift); + tcg_gen_andi_i64(t, t, TB_JMP_PAGE_MASK); + tcg_gen_shri_i64(h, pc, 2); + tcg_gen_shri_i64(v, pc, 6); + tcg_gen_xor_i64(h, h, v); + tcg_gen_andi_i64(h, h, TB_JMP_ADDR_MASK); + tcg_gen_or_i64(h, h, t); + tcg_gen_shli_i64(h, h, 4); + + /* the entry: tb, pc */ + tcg_gen_ld_ptr(jc, tcg_env, offsetof(CPUState, tb_jmp_cache) - sizeof(CPUState)); + tcg_gen_trunc_i64_ptr(ent, h); + tcg_gen_add_ptr(ent, ent, jc); + tcg_gen_ld_ptr(tbp, ent, offsetof(CPUJumpCache, array[0].tb)); + tcg_gen_ld_i64(t, ent, offsetof(CPUJumpCache, array[0].pc)); + tcg_gen_brcondi_ptr(TCG_COND_EQ, tbp, 0, slow); + tcg_gen_brcond_i64(TCG_COND_NE, t, pc, slow); + + /* the TB's state: cs_base, flags, cflags */ + tcg_gen_ld_i64(t, tbp, offsetof(TranslationBlock, cs_base)); + tcg_gen_xori_i64(t, t, tb->cs_base); + tcg_gen_ld32u_i64(v, tbp, offsetof(TranslationBlock, flags)); + tcg_gen_xori_i64(v, v, tb->flags); + tcg_gen_or_i64(t, t, v); + tcg_gen_ld32u_i64(v, tbp, offsetof(TranslationBlock, cflags)); + tcg_gen_xori_i64(v, v, tb_cflags(tb)); + tcg_gen_or_i64(t, t, v); + tcg_gen_brcondi_i64(TCG_COND_NE, t, 0, slow); + + /* no breakpoints (they need the C path's checks) */ + tcg_gen_ld_ptr(jc, tcg_env, offsetof(CPUState, breakpoints.tqh_first) - sizeof(CPUState)); + tcg_gen_brcondi_ptr(TCG_COND_NE, jc, 0, slow); + + tcg_gen_ld_ptr(code, tbp, offsetof(TranslationBlock, tc.ptr)); + tcg_gen_goto_ptr(code); + + /* A miss: the slow lookup, with the pc and the TB state we already have. */ + gen_set_label(slow); + gen_helper_lookup_tb_ptr_miss(code, tcg_env, pc, tcg_constant_i64(tb->cs_base), tcg_constant_i32(tb->flags)); + tcg_gen_goto_ptr(code); +} + bool translator_is_same_page(const DisasContextBase* db, vaddr addr) { return ((addr ^ db->pc_first) & TARGET_PAGE_MASK) == 0; } diff --git a/hw/arm/tests/native_int_check.S b/hw/arm/tests/native_int_check.S index 78ab1e893e..ea491d976a 100644 --- a/hw/arm/tests/native_int_check.S +++ b/hw/arm/tests/native_int_check.S @@ -16,6 +16,10 @@ * 8 SIMD&FP loads in a region: a Q load and a pair that cross a 16 KiB * page (slow path), and a Q load that faults: the destination stays * unchanged and the registers written before it are exact + * 9 an indirect branch to code whose TB for EL1 is in the jump cache + * (el_probe, called with BLR at EL1 first): at EL0 the lookup must + * not take the EL1 TB (the inline jump-cache probe checks the TB + * state, not just the pc), so its MRS TPIDR_EL1 traps * 7 MRS TPIDR_EL0 / TPIDRRO_EL0 (plain field reads) in the middle of * ALU code, after MSR TPIDR_EL0 at EL0 and TPIDRRO_EL0 set at EL1 * 6 a store over code later in the same block (the store's slow @@ -134,6 +138,8 @@ _start: ldr x0, =0x7070707070707070 msr tpidrro_el0, x0 + adr x0, el_probe /* test 9: its EL1 TB into the jump cache */ + blr x0 ldr x0, =BUF msr sp_el0, x0 adr x0, el0_main @@ -156,7 +162,10 @@ el1_sync: cmp x1, #0x15 /* SVC */ b.eq el1_svc cmp x1, #0x24 /* data abort from EL0 */ + b.eq 1f + cmp x1, #0x00 /* UNDEF from EL0 (test 9) */ b.ne el1_fail +1: ldr x0, =SAVE stp x2, x3, [x0, #2 * 8] stp x4, x5, [x0, #4 * 8] @@ -338,6 +347,19 @@ fault8: CHKSAVE 31, HOLE + 0x200, 8 fmov x12, d3 CHK x12, 0xffffffffffffffff, 8 /* destination unchanged */ + /* 9: the EL0 lookup of el_probe must not reuse its EL1 TB */ + LI x27, SAVE + str xzr, [x27, #34 * 8] + adr x9, el_probe + blr x9 + CHKSAVE 34, 0x02000000, 9 /* ESR: EC 0 (UNDEF), IL */ + adr x26, el_probe + LI x27, SAVE + ldr x27, [x27, #32 * 8] /* ELR */ + cmp x27, x26 + b.eq 2f + svc #9 +2: /* 7: TLS register reads in the middle of a block */ LI x0, 0x0123456789abcdef @@ -354,6 +376,11 @@ fault8: svc #0 +/* test 9: MRS TPIDR_EL1 works at EL1 and traps at EL0 */ +el_probe: + mrs x0, tpidr_el1 + ret + /* copied to CODE; x11 = the new instruction, x12 = where it goes */ smc_snip: movz x10, #0x1111 diff --git a/hw/arm/tests/native_int_gen.py b/hw/arm/tests/native_int_gen.py index 9123b20de9..a973ba1247 100644 --- a/hw/arm/tests/native_int_gen.py +++ b/hw/arm/tests/native_int_gen.py @@ -17,7 +17,8 @@ # Register use: x0-x17, x19-x21 random; x22 scratch for the hash; x23 hash; # x24 code anchor (ADR/ADRP results are made relative to it); x25 index # scratch; x26-x28 buffer pointers (moved by pre/post-index forms within -# their windows); x18, x29, x30 and sp unused (platform register, frame). +# their windows); x29 keeps nic_body's return address while x30 serves BLR/RET; +# x18 and sp unused (platform register, stack). # # SPDX-License-Identifier: AGPL-3.0-or-later @@ -381,6 +382,26 @@ def branch(self): self.ldst(wb=False) self.out.append(f"9{n}:") + def indirect(self): + """BR over a few instructions, or BLR to a local function that RETs (x30 is free: x29 keeps nic_body's).""" + r = self.r + self.label += 1 + n = self.label + if r.random() < 0.5: + self.emit(f"adr x25, 9{n}f") + self.emit("br x25") + for _ in range(r.randint(0, 2)): + self.alu() + else: + self.emit(f"adr x25, 7{n}f") + self.emit("blr x25") + self.emit(f"b 9{n}f") + self.out.append(f"7{n}:") + for _ in range(r.randint(1, 4)): + self.alu() + self.emit("ret") + self.out.append(f"9{n}:") + def fold(self): for n in self.r.sample(POOL, 6): self.emit(f"eor x23, x23, x{n}") @@ -388,6 +409,7 @@ def fold(self): def program(self, n): r = self.r + self.out.append(" mov x29, x30") self.out.append(" adr x24, nic_body") self.movimm(23, r.getrandbits(64)) for b in (26, 27, 28): @@ -406,8 +428,10 @@ def program(self, n): self.alu() elif p < 0.85: self.ldst() - elif p < 0.93: + elif p < 0.90: self.branch() + elif p < 0.93: + self.indirect() else: self.other() i += 1 @@ -427,6 +451,7 @@ def program(self, n): # buffer base back from x26's final offset, for nic_hash_buffer self.movimm(25, BASES[26][0] + self.off[26]) self.emit("sub x0, x26, x25") + self.emit("mov x30, x29") return "\n".join(self.out) + "\n" diff --git a/include/exec/translator.h b/include/exec/translator.h index 2ace45bbef..209b94778d 100644 --- a/include/exec/translator.h +++ b/include/exec/translator.h @@ -159,6 +159,20 @@ void translator_loop(CPUState* cpu, TranslationBlock* tb, int* max_insns, vaddr */ bool translator_use_goto_tb(DisasContextBase* db, vaddr dest); +/** + * translator_lookup_and_goto_ptr + * @db: disassembly context + * @pc: the guest pc to continue at + * @gen_lookup: the target's lookup helper, as for tcg_gen_lookup_and_goto_ptr_via() + * + * An indirect jump to the TB for @pc with the jump-cache hit inline. Only for + * exits where the CPU's TB state (flags, cs_base) is still that of @db->tb. + */ +struct TCGv_i64_d; +struct TCGv_ptr_d; +void translator_lookup_and_goto_ptr(DisasContextBase* db, struct TCGv_i64_d* pc, + void (*gen_lookup)(struct TCGv_ptr_d*, struct TCGv_ptr_d*)); + /** * translator_io_start * @db: Disassembly context diff --git a/include/tcg/tcg-op-common.h b/include/tcg/tcg-op-common.h index f218fd0393..ee09f9dc44 100644 --- a/include/tcg/tcg-op-common.h +++ b/include/tcg/tcg-op-common.h @@ -94,6 +94,8 @@ void tcg_gen_lookup_and_goto_ptr(void); * inline and then calls tcg_lookup_tb_ptr()) */ void tcg_gen_lookup_and_goto_ptr_via(void (*gen_lookup)(TCGv_ptr ret, TCGv_env env)); +/* Jump to host code at @ptr (a TB's tc.ptr, or tcg_code_gen_epilogue): ends the TB. */ +void tcg_gen_goto_ptr(TCGv_ptr ptr); /* 32 bit ops */ diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c index 2b1d6e5777..fcaecbeab3 100644 --- a/target/arm/tcg/translate-a64.c +++ b/target/arm/tcg/translate-a64.c @@ -397,6 +397,19 @@ static inline bool use_goto_tb(DisasContext* s, uint64_t dest) return translator_use_goto_tb(&s->base, dest); } +/* + * An indirect branch (or a direct one off this page) to the pc in cpu_pc, + * with the jump-cache hit inline. The TB state here is this TB's, except + * where BTI makes BTYPE (part of the state) depend on the branch. + */ +static void gen_a64_lookup_and_goto_ptr(DisasContext* s) +{ + if (dc_isar_feature(aa64_bti, s)) { gen_lookup_and_goto_ptr(); } + else { + translator_lookup_and_goto_ptr(&s->base, cpu_pc, gen_helper_arm_lookup_tb_ptr); + } +} + static void gen_goto_tb(DisasContext* s, int n, int64_t diff) { if (use_goto_tb(s, s->pc_curr + diff)) { @@ -423,7 +436,7 @@ static void gen_goto_tb(DisasContext* s, int n, int64_t diff) gen_a64_update_pc(s, diff); if (s->ss_active) { gen_step_complete_exception(s); } else { - gen_lookup_and_goto_ptr(); + gen_a64_lookup_and_goto_ptr(s); s->base.is_jmp = DISAS_NORETURN; } } @@ -9642,9 +9655,11 @@ static void aarch64_tr_tb_stop(DisasContextBase* dcbase, CPUState* cpu) /* fall through */ case DISAS_EXIT: tcg_gen_exit_tb(NULL, 0); break; case DISAS_UPDATE_NOCHAIN: + /* the TB state may have changed: the helper recomputes it */ gen_a64_update_pc(dc, 4); - /* fall through */ - case DISAS_JUMP : gen_lookup_and_goto_ptr(); break; + gen_lookup_and_goto_ptr(); + break; + case DISAS_JUMP: gen_a64_lookup_and_goto_ptr(dc); break; case DISAS_NORETURN: case DISAS_SWI : break; case DISAS_WFE: diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index a96f2e4352..dace920deb 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -2986,6 +2986,8 @@ void tcg_gen_goto_tb(unsigned idx) tcg_gen_op1i(INDEX_op_goto_tb, 0, idx); } +void tcg_gen_goto_ptr(TCGv_ptr ptr) { tcg_gen_op1i(INDEX_op_goto_ptr, TCG_TYPE_PTR, tcgv_ptr_arg(ptr)); } + void tcg_gen_lookup_and_goto_ptr_via(void (*gen_lookup)(TCGv_ptr ret, TCGv_env env)) { TCGv_ptr ptr;