From 00730d09726ddadb904de5123c5e58dbb24006d0 Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 5 Aug 2026 11:56:07 +0000
Subject: [PATCH] Improve SVE2 SynetInnerProduct16bGemmNN with svbfdot_f32

Replace BF16-to-FP32 unpack + svmla in the GemmNN microkernel with
direct BF16 accumulation via svbfdot_f32, matching the packed Kx2
layout already used for B.

Co-authored-by: Ihar Yermalayeu <ermig1979@gmail.com>
---
 docs/2026.html                                |   4 +
 .../SimdSve2SynetInnerProduct16bGemmNN.cpp    | 100 +++++++-----------
 2 files changed, 41 insertions(+), 63 deletions(-)

diff --git a/docs/2026.html b/docs/2026.html
index bdb4acde1c..108354f220 100644
--- a/docs/2026.html
+++ b/docs/2026.html
@@ -54,6 +54,10 @@ <h5>New features</h5>
  <li>SVE2 optimizations of class SynetInnerProduct16bGemmNN.</li>
  <li>NEON optimizations of class SynetInnerProduct16bGemmNN.</li>
 </ul>
+<h5>Improving</h5>
+<ul>
+ <li>SVE2 optimizations of class SynetInnerProduct16bGemmNN (use of svbfdot_f32).</li>
+</ul>
 <h5>Renaming</h5>
 <ul>
  <li>Rename function SimdVectorProduct to SimdCreateMask.</li>
diff --git a/src/Simd/SimdSve2SynetInnerProduct16bGemmNN.cpp b/src/Simd/SimdSve2SynetInnerProduct16bGemmNN.cpp
index d3153d527a..a0222c603f 100644
--- a/src/Simd/SimdSve2SynetInnerProduct16bGemmNN.cpp
+++ b/src/Simd/SimdSve2SynetInnerProduct16bGemmNN.cpp
@@ -49,9 +49,14 @@ namespace Simd
             return svlsr_n_u32_x(mask, svadd_u32_x(mask, bits, round), Base::Bf16::SHIFT);
         }
 
-        SIMD_INLINE svfloat32_t BroadcastBf16(uint16_t value)
+        SIMD_INLINE svbfloat16_t BroadcastBf16x2(const uint16_t* src)
         {
-            return svreinterpret_f32_u32(svdup_n_u32(uint32_t(value) << Base::Bf16::SHIFT));
+            return svreinterpret_bf16_u32(svdup_n_u32(uint32_t(src[0]) | (uint32_t(src[1]) << 16)));
+        }
+
+        SIMD_INLINE svbfloat16_t LoadBf16x2(const uint16_t* src, const svbool_t& mask)
+        {
+            return svreinterpret_bf16_u32(svld1_u32(mask, (const uint32_t*)src));
         }
 
         //-----------------------------------------------------------------------------------------
@@ -373,8 +378,8 @@ namespace Simd
         {
             const size_t F = a.F, DF = F * 2;
             const svbool_t body = svptrue_b32();
-            svfloat32_t c00, c01, c10, c11, c20, c21, c30, c31, c40, c41, a0, b00, b01, b10, b11;
-            svuint32_t maskBits = svdup_n_u32(Base::Bf16::MASK);
+            svfloat32_t c00, c01, c10, c11, c20, c21, c30, c31, c40, c41;
+            svbfloat16_t a0, b0, b1;
             size_t dC = a.cN, dA = a.aK, dD = p.N * a.eC;
             const uint16_t* B1 = B0 + a.bK * F;
             const uint16_t* A1 = A0 + 1 * dA;
@@ -401,56 +406,37 @@ namespace Simd
                 }
                 for (size_t k = 0; k < K; k += 2)
                 {
-                    svuint32_t b0u = svld1_u32(body, (const uint32_t*)B0);
-                    b00 = svreinterpret_f32_u32(svlsl_n_u32_x(body, b0u, Base::Bf16::SHIFT));
-                    b01 = svreinterpret_f32_u32(svand_u32_x(body, b0u, maskBits));
-                    svuint32_t b1u = svld1_u32(body, (const uint32_t*)B1);
-                    b10 = svreinterpret_f32_u32(svlsl_n_u32_x(body, b1u, Base::Bf16::SHIFT));
-                    b11 = svreinterpret_f32_u32(svand_u32_x(body, b1u, maskBits));
+                    b0 = LoadBf16x2(B0, body);
+                    b1 = LoadBf16x2(B1, body);
                     if (M > 0)
                     {
-                        a0 = BroadcastBf16(A0[k + 0]);
-                        c00 = svmla_f32_x(body, c00, a0, b00);
-                        c01 = svmla_f32_x(body, c01, a0, b10);
-                        a0 = BroadcastBf16(A0[k + 1]);
-                        c00 = svmla_f32_x(body, c00, a0, b01);
-                        c01 = svmla_f32_x(body, c01, a0, b11);
+                        a0 = BroadcastBf16x2(A0 + k);
+                        c00 = svbfdot_f32(c00, a0, b0);
+                        c01 = svbfdot_f32(c01, a0, b1);
                     }
                     if (M > 1)
                     {
-                        a0 = BroadcastBf16(A1[k + 0]);
-                        c10 = svmla_f32_x(body, c10, a0, b00);
-                        c11 = svmla_f32_x(body, c11, a0, b10);
-                        a0 = BroadcastBf16(A1[k + 1]);
-                        c10 = svmla_f32_x(body, c10, a0, b01);
-                        c11 = svmla_f32_x(body, c11, a0, b11);
+                        a0 = BroadcastBf16x2(A1 + k);
+                        c10 = svbfdot_f32(c10, a0, b0);
+                        c11 = svbfdot_f32(c11, a0, b1);
                     }
                     if (M > 2)
                     {
-                        a0 = BroadcastBf16(A2[k + 0]);
-                        c20 = svmla_f32_x(body, c20, a0, b00);
-                        c21 = svmla_f32_x(body, c21, a0, b10);
-                        a0 = BroadcastBf16(A2[k + 1]);
-                        c20 = svmla_f32_x(body, c20, a0, b01);
-                        c21 = svmla_f32_x(body, c21, a0, b11);
+                        a0 = BroadcastBf16x2(A2 + k);
+                        c20 = svbfdot_f32(c20, a0, b0);
+                        c21 = svbfdot_f32(c21, a0, b1);
                     }
                     if (M > 3)
                     {
-                        a0 = BroadcastBf16(A3[k + 0]);
-                        c30 = svmla_f32_x(body, c30, a0, b00);
-                        c31 = svmla_f32_x(body, c31, a0, b10);
-                        a0 = BroadcastBf16(A3[k + 1]);
-                        c30 = svmla_f32_x(body, c30, a0, b01);
-                        c31 = svmla_f32_x(body, c31, a0, b11);
+                        a0 = BroadcastBf16x2(A3 + k);
+                        c30 = svbfdot_f32(c30, a0, b0);
+                        c31 = svbfdot_f32(c31, a0, b1);
                     }
                     if (M > 4)
                     {
-                        a0 = BroadcastBf16(A4[k + 0]);
-                        c40 = svmla_f32_x(body, c40, a0, b00);
-                        c41 = svmla_f32_x(body, c41, a0, b10);
-                        a0 = BroadcastBf16(A4[k + 1]);
-                        c40 = svmla_f32_x(body, c40, a0, b01);
-                        c41 = svmla_f32_x(body, c41, a0, b11);
+                        a0 = BroadcastBf16x2(A4 + k);
+                        c40 = svbfdot_f32(c40, a0, b0);
+                        c41 = svbfdot_f32(c41, a0, b1);
                     }
                     B0 += DF;
                     B1 += DF;
@@ -482,43 +468,31 @@ namespace Simd
                 }
                 for (size_t k = 0; k < K; k += 2)
                 {
-                    svuint32_t b0u = svld1_u32(body, (const uint32_t*)B0);
-                    b00 = svreinterpret_f32_u32(svlsl_n_u32_x(body, b0u, Base::Bf16::SHIFT));
-                    b01 = svreinterpret_f32_u32(svand_u32_x(body, b0u, maskBits));
+                    b0 = LoadBf16x2(B0, body);
                     if (M > 0)
                     {
-                        a0 = BroadcastBf16(A0[k + 0]);
-                        c00 = svmla_f32_x(body, c00, a0, b00);
-                        a0 = BroadcastBf16(A0[k + 1]);
-                        c00 = svmla_f32_x(body, c00, a0, b01);
+                        a0 = BroadcastBf16x2(A0 + k);
+                        c00 = svbfdot_f32(c00, a0, b0);
                     }
                     if (M > 1)
                     {
-                        a0 = BroadcastBf16(A1[k + 0]);
-                        c10 = svmla_f32_x(body, c10, a0, b00);
-                        a0 = BroadcastBf16(A1[k + 1]);
-                        c10 = svmla_f32_x(body, c10, a0, b01);
+                        a0 = BroadcastBf16x2(A1 + k);
+                        c10 = svbfdot_f32(c10, a0, b0);
                     }
                     if (M > 2)
                     {
-                        a0 = BroadcastBf16(A2[k + 0]);
-                        c20 = svmla_f32_x(body, c20, a0, b00);
-                        a0 = BroadcastBf16(A2[k + 1]);
-                        c20 = svmla_f32_x(body, c20, a0, b01);
+                        a0 = BroadcastBf16x2(A2 + k);
+                        c20 = svbfdot_f32(c20, a0, b0);
                     }
                     if (M > 3)
                     {
-                        a0 = BroadcastBf16(A3[k + 0]);
-                        c30 = svmla_f32_x(body, c30, a0, b00);
-                        a0 = BroadcastBf16(A3[k + 1]);
-                        c30 = svmla_f32_x(body, c30, a0, b01);
+                        a0 = BroadcastBf16x2(A3 + k);
+                        c30 = svbfdot_f32(c30, a0, b0);
                     }
                     if (M > 4)
                     {
-                        a0 = BroadcastBf16(A4[k + 0]);
-                        c40 = svmla_f32_x(body, c40, a0, b00);
-                        a0 = BroadcastBf16(A4[k + 1]);
-                        c40 = svmla_f32_x(body, c40, a0, b01);
+                        a0 = BroadcastBf16x2(A4 + k);
+                        c40 = svbfdot_f32(c40, a0, b0);
                     }
                     B0 += DF;
                 }