target-arm: AArch32 Neon integer data processing and loads and stores

crates/target-arm/build.rs

@@ -1,10 +1,11 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 
 //! Generates the A64 decoders from QEMU's a64.decode and sve.decode, and the AArch32 ones from
-//! vfp.decode, vfp-uncond.decode, a32.decode, a32-uncond.decode, t32.decode and t16.decode,
-//! run with the options target/arm/tcg's meson.build passes (`--static-decode=disas_a64`,
-//! `--decode=disas_sve`, `--decode=disas_vfp`, `--decode=disas_vfp_uncond`,
-//! `--static-decode=disas_a32`, `--static-decode=disas_a32_uncond`,
+//! vfp.decode, vfp-uncond.decode, neon-dp.decode, neon-ls.decode, a32.decode,
+//! a32-uncond.decode, t32.decode and t16.decode, run with the options target/arm/tcg's
+//! meson.build passes (`--static-decode=disas_a64`, `--decode=disas_sve`,
+//! `--decode=disas_vfp`, `--decode=disas_vfp_uncond`, `--decode=disas_neon_dp`,
+//! `--decode=disas_neon_ls`, `--static-decode=disas_a32`, `--static-decode=disas_a32_uncond`,
 //! `--static-decode=disas_t32` and `--static-decode=disas_t16 --insnwidth=16`).
 //!
 //! ruvm-decode makes every `trans_` method a required trait method. Not every pattern is
@@ -32,6 +33,8 @@ fn main() {
         ("sve.decode", "--decode=disas_sve", "sve_decode.rs"),
         ("vfp.decode", "--decode=disas_vfp", "vfp_decode.rs"),
         ("vfp-uncond.decode", "--decode=disas_vfp_uncond", "vfp_uncond_decode.rs"),
+        ("neon-dp.decode", "--decode=disas_neon_dp", "neon_dp_decode.rs"),
+        ("neon-ls.decode", "--decode=disas_neon_ls", "neon_ls_decode.rs"),
     ] {
         let rust = generate(&manifest, name, &[opt]);
         write_if_changed(&dir.join(out), &default_trans(&rust));

crates/target-arm/src/tcg/a32_neon.rs

@@ -0,0 +1,126 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+//! The AArch32 Advanced SIMD integer helper: the operations of QEMU's `neon_helper.c` and of
+//! the gvec expansions that `translate-neon.c` uses, run on D register images by the A64
+//! helper [`vec_helper::neon`] where the two instruction sets agree.
+//!
+//! D`n` is half of the AdvSIMD register Q`n/2`, so the two D registers of a Q register are
+//! contiguous but D`2k+1` and D`2k+2` are not. The helper reads each operand as the 16 bytes
+//! of D`r` and D`r+1` and writes 8 or 16 bytes back the same way, leaving the rest of the
+//! SVE register alone as QEMU's AArch32 code does.
+
+use ruvm_jit_core::HelperType::{I32, Ptr, Void};
+use ruvm_jit_interp::{HelperEnv, Unwind};
+
+use super::helpers::{Def, def};
+use super::vec_helper::{self, Desc, Regs, V, op, set_qc};
+use crate::cpu::vreg_off;
+
+/// Write the result to D`d` and D`d+1`, not only D`d`.
+pub(crate) const OUT_Q: u32 = 1 << 20;
+/// Also run operation `op + 1` and write its result to D`m` (and D`m+1`), for VTRN, VUZP
+/// and VZIP.
+pub(crate) const PAIR: u32 = 1 << 21;
+
+def!(NEON, "a32_neon", 0, Void, [Ptr, I32, I32], h_neon);
+
+/// The helpers of this module.
+pub(crate) const ALL: &[Def] = &[NEON];
+
+/// The `env` offset of D`r`.
+fn d_off(r: u32) -> usize {
+    let r = r as usize;
+    vreg_off(r >> 1) + 8 * (r & 1)
+}
+
+/// D`r` and D`r+1` as one image.
+fn dload(env: &[u8], r: u32) -> V {
+    let mut v = [0; 16];
+    let (lo, hi) = (d_off(r), d_off(r + 1));
+    v[..8].copy_from_slice(&env[lo..lo + 8]);
+    v[8..].copy_from_slice(&env[hi..hi + 8]);
+    v
+}
+
+/// Write the low 8 bytes of `v` to D`r`, and the high 8 bytes to D`r+1` if `q`.
+fn dstore(env: &mut [u8], r: u32, v: &V, q: bool) {
+    let lo = d_off(r);
+    env[lo..lo + 8].copy_from_slice(&v[..8]);
+    if q {
+        let hi = d_off(r + 1);
+        env[hi..hi + 8].copy_from_slice(&v[8..]);
+    }
+}
+
+/// `a32_neon(env, regs, desc)`: the operation in `desc` (a [`Desc`]) on D`n`, D`m` and the
+/// old D`d` in `regs` (from [`Regs::pack`], with [`OUT_Q`] and [`PAIR`]).
+fn h_neon(h: &mut HelperEnv<'_>, a: &[u64]) -> Result<u128, Unwind> {
+    let r = Regs::unpack(a[1]);
+    let flags = a[1] as u32;
+    let q = flags & OUT_Q != 0;
+    let desc = Desc::unpack(a[2]);
+    let env = &mut *h.env;
+    let n = dload(env, r.n);
+    let m = dload(env, r.m);
+    let d = dload(env, r.d);
+    let mut sat = false;
+    let out = if matches!(desc.op, op::TBL | op::TBX) {
+        vtbl(env, &desc, r.n, &m, &d)
+    } else {
+        vec_helper::neon(&desc, r, env, &n, &m, &d, &mut sat)
+    };
+    if flags & PAIR != 0 {
+        let desc2 = Desc { op: desc.op + 1, ..desc };
+        let out2 = vec_helper::neon(&desc2, r, env, &n, &m, &d, &mut sat);
+        // QEMU writes Dm first, so Dd wins when they are the same register, except for
+        // VTRN.32, which swaps the elements one at a time.
+        dstore(env, r.m, &out2, q);
+        if r.d == r.m && desc.op == op::TRN1 && desc.esz == 2 {
+            let rev = Desc { op: op::REV64, ..desc };
+            let out = vec_helper::neon(&rev, r, env, &d, &d, &d, &mut sat);
+            dstore(env, r.d, &out, q);
+        } else {
+            dstore(env, r.d, &out, q);
+        }
+    } else {
+        dstore(env, r.d, &out, q);
+    }
+    if sat {
+        set_qc(env);
+    }
+    Ok(0)
+}
+
+/// VTBL and VTBX: the table is the `imm + 1` registers from D`n`, the indexes the bytes of
+/// D`m`. An index past the table gives 0, or the old byte of D`d` for VTBX.
+fn vtbl(env: &[u8], desc: &Desc, rn: u32, m: &V, d: &V) -> V {
+    let len = desc.imm as usize + 1;
+    let mut table = [0u8; 32];
+    for k in 0..len {
+        let o = d_off(rn + k as u32);
+        table[8 * k..8 * k + 8].copy_from_slice(&env[o..o + 8]);
+    }
+    let mut out = [0u8; 16];
+    for i in 0..8 {
+        let ix = m[i] as usize;
+        out[i] = if ix < 8 * len {
+            table[ix]
+        } else if desc.op == op::TBX {
+            d[i]
+        } else {
+            0
+        };
+    }
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn d_registers_follow_the_q_registers() {
+        assert_eq!(d_off(3), vreg_off(1) + 8);
+        assert_eq!(d_off(4), vreg_off(2));
+    }
+}

crates/target-arm/src/tcg/helpers.rs

@@ -179,6 +179,7 @@ pub(crate) fn register(r: &mut HelperRegistry) {
         super::a32_helper::PAR_GE,
         super::a32_helper::PAR,
         super::a32_vfp::ALL,
+        super::a32_neon::ALL,
     ];
     for d in lists.iter().copied().flatten() {
         r.register_info(&d.info(), d.f);

crates/target-arm/src/tcg/mod.rs

@@ -145,6 +145,7 @@
 //!   returns with `LOG_GUEST_ERROR`; this crate has no logging and stays silent.
 
 mod a32_helper;
+mod a32_neon;
 mod a32_vfp;
 mod crypto;
 mod gic;

crates/target-arm/src/tcg/translate.rs

@@ -71,6 +71,7 @@ mod simd;
 #[path = "translate_sve.rs"]
 mod sve;
 
+pub(super) use simd::asimd_imm_const;
 use simd::{Chk, Feat, Mov, RA, RF, RM, RN, RP, RZ, cop, fop, nop};
 
 use decode::{

crates/target-arm/src/tcg/translate_a32.rs

@@ -2,16 +2,16 @@
 
 //! The A32 and T32 translator: the integer part of QEMU's `target/arm/tcg/translate.c`.
 //!
-//! The decoders come from a32.decode, a32-uncond.decode, t32.decode, t16.decode, vfp.decode
-//! and vfp-uncond.decode (see `build.rs`). [`S`] implements their `trans_` hooks for the
-//! integer instructions, the barriers and the cp15 thread ID registers; the `fp` module has
-//! the VFP ones. Every pattern left out returns false, which raises an Undefined Instruction
+//! The decoders come from a32.decode, a32-uncond.decode, t32.decode, t16.decode, vfp.decode,
+//! vfp-uncond.decode, neon-dp.decode and neon-ls.decode (see `build.rs`). [`S`] implements
+//! their `trans_` hooks for the integer instructions, the barriers and the cp15 thread ID
+//! registers; the `fp` module has the VFP ones and the `neon` module the Advanced SIMD ones. Every pattern left out returns false, which raises an Undefined Instruction
 //! exception as QEMU's `unallocated_encoding()` does.
 //!
 //! As in QEMU, R0 to R14, the PC and the NZCV flags are TCG globals. R15 is the PC field,
 //! which holds the zero extended 32-bit PC in AArch32 state.
 //!
-//! Not here yet, and so UNDEF: Neon, the BF16 VCVT, VJCVT, the M profile, SWP, and what
+//! Not here yet, and so UNDEF: the FP and crypto parts of Neon, the BF16 VCVT, VJCVT, the M profile, SWP, and what
 //! only AArch32 system mode needs (exception returns, banked registers, SPSR, RFE, SRS,
 //! CPS, LDM and STM of the User registers, and the coprocessor registers other than
 //! TPIDRURW, TPIDRURO and the CP15 barriers). Only the user mode emulator runs AArch32 code so far, so the
@@ -24,6 +24,7 @@ use ruvm_jit_core::{Func, Label, MemOp, Temp};
 use ruvm_mem::Endian;
 
 mod fp;
+mod neon;
 
 use super::a32_helper as h;
 use super::helpers::{self, Def};
@@ -2482,7 +2483,11 @@ impl S<'_, '_> {
         }
         if cond == 0xf {
             // Unconditional instructions.
-            if !a32_uncond::disas(self, insn) && !fp::disas_uncond(self, insn) {
+            if !a32_uncond::disas(self, insn)
+                && !fp::disas_uncond(self, insn)
+                && !neon::disas_dp(self, insn)
+                && !neon::disas_ls(self, insn)
+            {
                 self.unallocated_encoding();
             }
             return;
@@ -2550,7 +2555,16 @@ impl DisasContext {
         let ok = if is_16bit {
             t16::disas(&mut s, insn as u16)
         } else {
-            t32::disas(&mut s, insn)
+            // The Neon data processing and load/store encodings, rewritten as their A32
+            // forms.
+            (insn & 0xef00_0000 == 0xef00_0000
+                && neon::disas_dp(
+                    &mut s,
+                    (insn & 0xe2ff_ffff) | ((insn & (1 << 28)) >> 4) | (1 << 28),
+                ))
+                || (insn & 0xff10_0000 == 0xf900_0000
+                    && neon::disas_ls(&mut s, (insn & 0x00ff_ffff) | 0xf400_0000))
+                || t32::disas(&mut s, insn)
                 || fp::disas_uncond(&mut s, insn)
                 || (insn >> 28 == 0xe && fp::disas(&mut s, insn))
         };

crates/target-arm/src/tcg/translate_a32/fp.rs

@@ -9,8 +9,7 @@
 //! through the one `a32_vfp` helper, with the values in 64-bit temporaries whatever their
 //! size.
 //!
-//! Not here yet, and so UNDEF: VCVT to BFloat16, VJCVT, and the Advanced SIMD scalar moves
-//! (VMOV to and from a scalar, VDUP).
+//! Not here yet, and so UNDEF: VCVT to BFloat16 and VJCVT.
 
 use ruvm_jit_core::MemOp;
 use ruvm_jit_core::ir::{TempI32, TempI64};
@@ -53,7 +52,7 @@ const fn rm(m: RoundMode) -> u32 {
 
 /// `vfp_reg_offset()`: the `env` offset of D`reg`, or of S`reg` if not `dp`. D`n` is half of
 /// the AdvSIMD register Q`n/2`, and S`n` half of D`n/2`.
-fn vfp_reg_offset(dp: bool, reg: i32) -> usize {
+pub(super) fn vfp_reg_offset(dp: bool, reg: i32) -> usize {
     let reg = reg as usize;
     if dp {
         vreg_off(reg >> 1) + 8 * (reg & 1)
@@ -411,6 +410,55 @@ impl DisasVfp for S<'_, '_> {
         true
     }
 
+    fn trans_VMOV_to_gp(&mut self, a: &mut arg_VMOV_to_gp) -> bool {
+        // VMOV scalar to general purpose register
+        let off = (vfp_reg_offset(true, a.vn) + ((a.index as usize) << a.size)) as i64;
+        let t = self.new32();
+        let env = self.env();
+        let f = self.f();
+        match (a.size, a.u != 0) {
+            (0, false) => f.gen_ld8s_i32(t, env, off),
+            (0, true) => f.gen_ld8u_i32(t, env, off),
+            (1, false) => f.gen_ld16s_i32(t, env, off),
+            (1, true) => f.gen_ld16u_i32(t, env, off),
+            _ => f.gen_ld_i32(t, env, off),
+        }
+        self.store_reg(a.rt, t);
+        true
+    }
+
+    fn trans_VMOV_from_gp(&mut self, a: &mut arg_VMOV_from_gp) -> bool {
+        // VMOV general purpose register to scalar
+        let off = (vfp_reg_offset(true, a.vn) + ((a.index as usize) << a.size)) as i64;
+        let t = self.load_reg(a.rt);
+        let env = self.env();
+        let f = self.f();
+        match a.size {
+            0 => f.gen_st8_i32(t, env, off),
+            1 => f.gen_st16_i32(t, env, off),
+            _ => f.gen_st_i32(t, env, off),
+        }
+        true
+    }
+
+    fn trans_VDUP(&mut self, a: &mut arg_VDUP) -> bool {
+        // VDUP (general purpose register)
+        if (a.b != 0 && a.e != 0) || (a.q != 0 && a.vn & 1 != 0) {
+            return false;
+        }
+        let size = if a.b != 0 {
+            0
+        } else if a.e != 0 {
+            1
+        } else {
+            2
+        };
+        let t = self.load_reg(a.rt);
+        let x = self.dup64(t, size);
+        self.st_dreg_q(x, a.vn, a.q != 0);
+        true
+    }
+
     fn trans_VMOV_single(&mut self, a: &mut arg_VMOV_single) -> bool {
         if a.l != 0 {
             // VFP to general purpose register