accel/tcg, target/arm: inline jump-cache probe for A64 indirect branches

accel/tcg/cpu-exec.c

@@ -313,6 +313,19 @@ const void* HELPER(lookup_tb_ptr)(CPUArchState* env)
     return lookup_tb_ptr(cpu, s.pc, s.cs_base, s.flags);
 }
 
+/*
+ * The miss path of translator_lookup_and_goto_ptr()'s inline jump-cache
+ * probe: the generated code has already looked in the jump cache and knows
+ * the TB state (that of the TB it belongs to), so go straight to the slow
+ * part.
+ */
+const void* HELPER(lookup_tb_ptr_miss)(CPUArchState* env, uint64_t pc, uint64_t cs_base, uint32_t flags)
+{
+    CPUState* cpu = env_cpu(env);
+
+    return tb_lookup_ptr_slow(cpu, pc, cs_base, flags, curr_cflags(cpu));
+}
+
 /* Return the current PC from CPU, which may be cached in TB. */
 static vaddr log_pc(CPUState* cpu, const TranslationBlock* tb)
 {

accel/tcg/internal-common.h

@@ -48,6 +48,9 @@ void tcg_exec_unrealizefn(CPUState* cpu);
  * Current cflags for hashing/comparison. Inline: it runs on every TB lookup,
  * including each indirect branch (helper_lookup_tb_ptr).
  */
+/* -accel tcg,x-inline-lookup=on|off (default on): see translator_lookup_and_goto_ptr() */
+extern bool tcg_inline_lookup;
+
 static inline uint32_t curr_cflags(CPUState* cpu)
 {
     uint32_t cflags = cpu->tcg_cflags;

accel/tcg/tcg-all.c

@@ -89,6 +89,7 @@ bool tcg_native_fp_explicit;
 bool tcg_native_fp_fault;
 bool tcg_native_fp_selftest = true;
 bool tcg_native_fp_inline   = true;
+bool tcg_inline_lookup = true;
 bool tcg_native_int;
 bool tcg_native_int_explicit;
 bool tcg_native_int_ldst = true;
@@ -296,6 +297,8 @@ static void tcg_set_native_fp_selftest(Object* obj, bool value, Error** errp) {
 static bool tcg_get_native_fp_inline(Object* obj, Error** errp) { return tcg_native_fp_inline; }
 static void tcg_set_native_fp_inline(Object* obj, bool value, Error** errp) { tcg_native_fp_inline = value; }
 
+static bool tcg_get_inline_lookup(Object* obj, Error** errp) { return tcg_inline_lookup; }
+static void tcg_set_inline_lookup(Object* obj, bool value, Error** errp) { tcg_inline_lookup = value; }
 static bool tcg_get_native_int(Object* obj, Error** errp) { return tcg_native_int; }
 static void tcg_set_native_int(Object* obj, bool value, Error** errp)
 {
@@ -358,6 +361,10 @@ static void tcg_accel_class_init(ObjectClass* oc, const void* data)
     object_class_property_set_description(oc, "x-native-fp-inline",
                                           "Diagnostic: set off to keep native-fp in helpers (no inline host FP)");
 
+    object_class_property_add_bool(oc, "x-inline-lookup", tcg_get_inline_lookup, tcg_set_inline_lookup);
+    object_class_property_set_description(oc, "x-inline-lookup",
+                                          "Diagnostic: set off to do every indirect-branch TB lookup in the helper");
+
     object_class_property_add_bool(oc, "native-int", tcg_get_native_int, tcg_set_native_int);
     object_class_property_set_description(oc, "native-int",
                                           "Execute guest EL0 integer code, loads/stores and branches as the same "

accel/tcg/tcg-runtime.h

@@ -25,6 +25,7 @@ DEF_HELPER_FLAGS_1(ctpop_i32, TCG_CALL_NO_RWG_SE, i32, i32)
 DEF_HELPER_FLAGS_1(ctpop_i64, TCG_CALL_NO_RWG_SE, i64, i64)
 
 DEF_HELPER_FLAGS_1(lookup_tb_ptr, TCG_CALL_NO_WG_SE, cptr, env)
+DEF_HELPER_FLAGS_4(lookup_tb_ptr_miss, TCG_CALL_NO_WG_SE, cptr, env, i64, i64, i32)
 
 DEF_HELPER_FLAGS_1(exit_atomic, TCG_CALL_NO_WG, noreturn, env)
 

accel/tcg/translator.c

@@ -18,6 +18,9 @@
 #include "tcg/tcg-op-common.h"
 #include "internal-common.h"
 #include "tb-internal.h"
+#include "tb-hash.h"
+#include "hw/core/cpu.h"
+#include "exec/helper-gen-common.h"
 
 bool translator_io_start(DisasContextBase* db)
 {
@@ -58,6 +61,92 @@ static void gen_tb_end(const TranslationBlock* tb, uint32_t cflags, int num_insn
     }
 }
 
+/*
+ * Look up the TB for the guest pc in @pc and jump to it, like
+ * tcg_gen_lookup_and_goto_ptr_via(@gen_lookup), but with the jump-cache hit
+ * done inline in the generated code: hash the pc, load the cache entry,
+ * check its pc and the TB's cs_base, flags and cflags against those of the
+ * TB being translated, check for breakpoints, and jump to its code. Anything
+ * else (a miss, another TB state, breakpoints) calls helper_lookup_tb_ptr_miss(),
+ * which skips the state computation and the jump-cache probe that @gen_lookup
+ * would repeat.
+ *
+ * Only valid when the CPU state at this exit is the state of the TB being
+ * translated (@db->tb flags and cs_base) and its cflags are the vCPU's
+ * current ones: the caller asserts the former; the latter is checked here.
+ * The jump-cache hit in C (lookup_tb_ptr()) costs a helper call and some 70
+ * host instructions; this is about 25. -accel tcg,x-inline-lookup=off keeps
+ * the helper for every lookup.
+ */
+void translator_lookup_and_goto_ptr(DisasContextBase* db, TCGv_i64 pc, void (*gen_lookup)(TCGv_ptr, TCGv_env))
+{
+    const TranslationBlock* tb = db->tb;
+    TCGLabel*               slow;
+    TCGv_ptr                jc, ent, tbp, code;
+    TCGv_i64                h, t, v;
+    int                     shift = TARGET_PAGE_BITS - TB_JMP_PAGE_BITS;
+
+    if (!tcg_inline_lookup || (tb_cflags(tb) & CF_NO_GOTO_PTR) || tb_cflags(tb) != curr_cflags(tcg_ctx->cpu) ||
+        qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) {
+        tcg_gen_lookup_and_goto_ptr_via(gen_lookup);
+        return;
+    }
+
+    slow = gen_new_label();
+    h    = tcg_temp_new_i64();
+    t    = tcg_temp_new_i64();
+    v    = tcg_temp_new_i64();
+    jc   = tcg_temp_new_ptr();
+    ent  = tcg_temp_new_ptr();
+    tbp  = tcg_temp_new_ptr();
+    code = tcg_temp_new_ptr();
+
+    /* h = tb_jmp_cache_hash_func(pc) */
+    QEMU_BUILD_BUG_ON(sizeof(((CPUJumpCache*)0)->array[0]) != 16);
+    tcg_gen_shri_i64(t, pc, shift);
+    tcg_gen_xor_i64(t, t, pc);
+    tcg_gen_shri_i64(t, t, shift);
+    tcg_gen_andi_i64(t, t, TB_JMP_PAGE_MASK);
+    tcg_gen_shri_i64(h, pc, 2);
+    tcg_gen_shri_i64(v, pc, 6);
+    tcg_gen_xor_i64(h, h, v);
+    tcg_gen_andi_i64(h, h, TB_JMP_ADDR_MASK);
+    tcg_gen_or_i64(h, h, t);
+    tcg_gen_shli_i64(h, h, 4);
+
+    /* the entry: tb, pc */
+    tcg_gen_ld_ptr(jc, tcg_env, offsetof(CPUState, tb_jmp_cache) - sizeof(CPUState));
+    tcg_gen_trunc_i64_ptr(ent, h);
+    tcg_gen_add_ptr(ent, ent, jc);
+    tcg_gen_ld_ptr(tbp, ent, offsetof(CPUJumpCache, array[0].tb));
+    tcg_gen_ld_i64(t, ent, offsetof(CPUJumpCache, array[0].pc));
+    tcg_gen_brcondi_ptr(TCG_COND_EQ, tbp, 0, slow);
+    tcg_gen_brcond_i64(TCG_COND_NE, t, pc, slow);
+
+    /* the TB's state: cs_base, flags, cflags */
+    tcg_gen_ld_i64(t, tbp, offsetof(TranslationBlock, cs_base));
+    tcg_gen_xori_i64(t, t, tb->cs_base);
+    tcg_gen_ld32u_i64(v, tbp, offsetof(TranslationBlock, flags));
+    tcg_gen_xori_i64(v, v, tb->flags);
+    tcg_gen_or_i64(t, t, v);
+    tcg_gen_ld32u_i64(v, tbp, offsetof(TranslationBlock, cflags));
+    tcg_gen_xori_i64(v, v, tb_cflags(tb));
+    tcg_gen_or_i64(t, t, v);
+    tcg_gen_brcondi_i64(TCG_COND_NE, t, 0, slow);
+
+    /* no breakpoints (they need the C path's checks) */
+    tcg_gen_ld_ptr(jc, tcg_env, offsetof(CPUState, breakpoints.tqh_first) - sizeof(CPUState));
+    tcg_gen_brcondi_ptr(TCG_COND_NE, jc, 0, slow);
+
+    tcg_gen_ld_ptr(code, tbp, offsetof(TranslationBlock, tc.ptr));
+    tcg_gen_goto_ptr(code);
+
+    /* A miss: the slow lookup, with the pc and the TB state we already have. */
+    gen_set_label(slow);
+    gen_helper_lookup_tb_ptr_miss(code, tcg_env, pc, tcg_constant_i64(tb->cs_base), tcg_constant_i32(tb->flags));
+    tcg_gen_goto_ptr(code);
+}
+
 bool translator_is_same_page(const DisasContextBase* db, vaddr addr)
 { return ((addr ^ db->pc_first) & TARGET_PAGE_MASK) == 0; }
 

hw/arm/tests/native_int_check.S

@@ -16,6 +16,10 @@
  *   8    SIMD&FP loads in a region: a Q load and a pair that cross a 16 KiB
  *        page (slow path), and a Q load that faults: the destination stays
  *        unchanged and the registers written before it are exact
+ *   9    an indirect branch to code whose TB for EL1 is in the jump cache
+ *        (el_probe, called with BLR at EL1 first): at EL0 the lookup must
+ *        not take the EL1 TB (the inline jump-cache probe checks the TB
+ *        state, not just the pc), so its MRS TPIDR_EL1 traps
  *   7    MRS TPIDR_EL0 / TPIDRRO_EL0 (plain field reads) in the middle of
  *        ALU code, after MSR TPIDR_EL0 at EL0 and TPIDRRO_EL0 set at EL1
  *   6    a store over code later in the same block (the store's slow
@@ -134,6 +138,8 @@ _start:
 
     ldr     x0, =0x7070707070707070
     msr     tpidrro_el0, x0
+    adr     x0, el_probe                        /* test 9: its EL1 TB into the jump cache */
+    blr     x0
     ldr     x0, =BUF
     msr     sp_el0, x0
     adr     x0, el0_main
@@ -156,7 +162,10 @@ el1_sync:
     cmp     x1, #0x15                           /* SVC */
     b.eq    el1_svc
     cmp     x1, #0x24                           /* data abort from EL0 */
+    b.eq    1f
+    cmp     x1, #0x00                           /* UNDEF from EL0 (test 9) */
     b.ne    el1_fail
+1:
     ldr     x0, =SAVE
     stp     x2, x3, [x0, #2 * 8]
     stp     x4, x5, [x0, #4 * 8]
@@ -338,6 +347,19 @@ fault8:
     CHKSAVE 31, HOLE + 0x200, 8
     fmov    x12, d3
     CHK     x12, 0xffffffffffffffff, 8          /* destination unchanged */
+    /* 9: the EL0 lookup of el_probe must not reuse its EL1 TB */
+    LI      x27, SAVE
+    str     xzr, [x27, #34 * 8]
+    adr     x9, el_probe
+    blr     x9
+    CHKSAVE 34, 0x02000000, 9                   /* ESR: EC 0 (UNDEF), IL */
+    adr     x26, el_probe
+    LI      x27, SAVE
+    ldr     x27, [x27, #32 * 8]                 /* ELR */
+    cmp     x27, x26
+    b.eq    2f
+    svc     #9
+2:
 
     /* 7: TLS register reads in the middle of a block */
     LI      x0, 0x0123456789abcdef
@@ -354,6 +376,11 @@ fault8:
 
     svc     #0
 
+/* test 9: MRS TPIDR_EL1 works at EL1 and traps at EL0 */
+el_probe:
+    mrs     x0, tpidr_el1
+    ret
+
 /* copied to CODE; x11 = the new instruction, x12 = where it goes */
 smc_snip:
     movz    x10, #0x1111

hw/arm/tests/native_int_gen.py

@@ -17,7 +17,8 @@
 # Register use: x0-x17, x19-x21 random; x22 scratch for the hash; x23 hash;
 # x24 code anchor (ADR/ADRP results are made relative to it); x25 index
 # scratch; x26-x28 buffer pointers (moved by pre/post-index forms within
-# their windows); x18, x29, x30 and sp unused (platform register, frame).
+# their windows); x29 keeps nic_body's return address while x30 serves BLR/RET;
+# x18 and sp unused (platform register, stack).
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
@@ -381,13 +382,34 @@ def branch(self):
                 self.ldst(wb=False)
         self.out.append(f"9{n}:")
 
+    def indirect(self):
+        """BR over a few instructions, or BLR to a local function that RETs (x30 is free: x29 keeps nic_body's)."""
+        r = self.r
+        self.label += 1
+        n = self.label
+        if r.random() < 0.5:
+            self.emit(f"adr x25, 9{n}f")
+            self.emit("br x25")
+            for _ in range(r.randint(0, 2)):
+                self.alu()
+        else:
+            self.emit(f"adr x25, 7{n}f")
+            self.emit("blr x25")
+            self.emit(f"b 9{n}f")
+            self.out.append(f"7{n}:")
+            for _ in range(r.randint(1, 4)):
+                self.alu()
+            self.emit("ret")
+        self.out.append(f"9{n}:")
+
     def fold(self):
         for n in self.r.sample(POOL, 6):
             self.emit(f"eor x23, x23, x{n}")
             self.emit("ror x23, x23, #11")
 
     def program(self, n):
         r = self.r
+        self.out.append("    mov x29, x30")
         self.out.append("    adr x24, nic_body")
         self.movimm(23, r.getrandbits(64))
         for b in (26, 27, 28):
@@ -406,8 +428,10 @@ def program(self, n):
                 self.alu()
             elif p < 0.85:
                 self.ldst()
-            elif p < 0.93:
+            elif p < 0.90:
                 self.branch()
+            elif p < 0.93:
+                self.indirect()
             else:
                 self.other()
             i += 1
@@ -427,6 +451,7 @@ def program(self, n):
         # buffer base back from x26's final offset, for nic_hash_buffer
         self.movimm(25, BASES[26][0] + self.off[26])
         self.emit("sub x0, x26, x25")
+        self.emit("mov x30, x29")
         return "\n".join(self.out) + "\n"
 
 

include/exec/translator.h

@@ -159,6 +159,20 @@ void translator_loop(CPUState* cpu, TranslationBlock* tb, int* max_insns, vaddr
  */
 bool translator_use_goto_tb(DisasContextBase* db, vaddr dest);
 
+/**
+ * translator_lookup_and_goto_ptr
+ * @db: disassembly context
+ * @pc: the guest pc to continue at
+ * @gen_lookup: the target's lookup helper, as for tcg_gen_lookup_and_goto_ptr_via()
+ *
+ * An indirect jump to the TB for @pc with the jump-cache hit inline. Only for
+ * exits where the CPU's TB state (flags, cs_base) is still that of @db->tb.
+ */
+struct TCGv_i64_d;
+struct TCGv_ptr_d;
+void translator_lookup_and_goto_ptr(DisasContextBase* db, struct TCGv_i64_d* pc,
+                                    void (*gen_lookup)(struct TCGv_ptr_d*, struct TCGv_ptr_d*));
+
 /**
  * translator_io_start
  * @db: Disassembly context

include/tcg/tcg-op-common.h

@@ -94,6 +94,8 @@ void tcg_gen_lookup_and_goto_ptr(void);
  *              inline and then calls tcg_lookup_tb_ptr())
  */
 void tcg_gen_lookup_and_goto_ptr_via(void (*gen_lookup)(TCGv_ptr ret, TCGv_env env));
+/* Jump to host code at @ptr (a TB's tc.ptr, or tcg_code_gen_epilogue): ends the TB. */
+void tcg_gen_goto_ptr(TCGv_ptr ptr);
 
 /* 32 bit ops */
 

target/arm/tcg/translate-a64.c

@@ -397,6 +397,19 @@ static inline bool use_goto_tb(DisasContext* s, uint64_t dest)
     return translator_use_goto_tb(&s->base, dest);
 }
 
+/*
+ * An indirect branch (or a direct one off this page) to the pc in cpu_pc,
+ * with the jump-cache hit inline. The TB state here is this TB's, except
+ * where BTI makes BTYPE (part of the state) depend on the branch.
+ */
+static void gen_a64_lookup_and_goto_ptr(DisasContext* s)
+{
+    if (dc_isar_feature(aa64_bti, s)) { gen_lookup_and_goto_ptr(); }
+    else {
+        translator_lookup_and_goto_ptr(&s->base, cpu_pc, gen_helper_arm_lookup_tb_ptr);
+    }
+}
+
 static void gen_goto_tb(DisasContext* s, int n, int64_t diff)
 {
     if (use_goto_tb(s, s->pc_curr + diff)) {
@@ -423,7 +436,7 @@ static void gen_goto_tb(DisasContext* s, int n, int64_t diff)
         gen_a64_update_pc(s, diff);
         if (s->ss_active) { gen_step_complete_exception(s); }
         else {
-            gen_lookup_and_goto_ptr();
+            gen_a64_lookup_and_goto_ptr(s);
             s->base.is_jmp = DISAS_NORETURN;
         }
     }
@@ -9642,9 +9655,11 @@ static void aarch64_tr_tb_stop(DisasContextBase* dcbase, CPUState* cpu)
                 /* fall through */
             case DISAS_EXIT: tcg_gen_exit_tb(NULL, 0); break;
             case DISAS_UPDATE_NOCHAIN:
+                /* the TB state may have changed: the helper recomputes it */
                 gen_a64_update_pc(dc, 4);
-                /* fall through */
-            case DISAS_JUMP    : gen_lookup_and_goto_ptr(); break;
+                gen_lookup_and_goto_ptr();
+                break;
+            case DISAS_JUMP: gen_a64_lookup_and_goto_ptr(dc); break;
             case DISAS_NORETURN:
             case DISAS_SWI     : break;
             case DISAS_WFE:

tcg/tcg-op.c

@@ -2986,6 +2986,8 @@ void tcg_gen_goto_tb(unsigned idx)
     tcg_gen_op1i(INDEX_op_goto_tb, 0, idx);
 }
 
+void tcg_gen_goto_ptr(TCGv_ptr ptr) { tcg_gen_op1i(INDEX_op_goto_ptr, TCG_TYPE_PTR, tcgv_ptr_arg(ptr)); }
+
 void tcg_gen_lookup_and_goto_ptr_via(void (*gen_lookup)(TCGv_ptr ret, TCGv_env env))
 {
     TCGv_ptr ptr;