tenstorrent: f32-out matmul must not land on the bf16 grid; the kolibri residue is chain drift through a high-gain router

.agents/issues/MODEL-TEXT-kolibri-1-tenstorrent/ISSUE-LOCAL-01M4ER0E9HHM95YZYJB7T5FECN.md

@@ -155,3 +155,46 @@ B2b-i landed the dense-resident device forward with the routed-expert tier delib
   slot-pool `what()=="1"` throw did NOT fire on the fixed substrate in any
   instrument or gate run (0 occurrences). The remaining flips after the fix
   are the residual bf16 device-arm drift class, not the byte bug.
+
+- 2026-10-10 (branch row/tt-kolibri-l1moe, off 519c466b8): the L1-moe ~3%
+  relative divergence is LOCALIZED and CLASSIFIED as the drift class, not a
+  stride/staging bug — instrument on the fixed substrate, teacher-forced
+  'der Mond...' (VT_TF_DBG_PROMPT=1, 12 forwards), OFF-vs-CPU with per-stage
+  bf16 bit hashes (StageDump hash=/sq=), the router dump (RouteDump, both
+  arms), the router weight hash probe, host-reference logits for BOTH arms'
+  router GEMMs, and the element-level L0 dhn diff. Evidence chain: (1) the
+  FIRST diverging quantity is the L0 input rms_norm output at step 0
+  (hash b3eeb298 vs 2b64e8d9; element diff 1 bf16 ULP class, top ~0.03-0.06
+  on values to 14, rms 0.002) — the adjudicated substrate drift, not new;
+  (2) by L0 dh2 (the moe input) the drift carries OUTLIER element diffs
+  (max 47.25 vs 48.0, 3 bf16 ULP on a 48-magnitude element; sq 2.166e4 vs
+  2.175e4, ~0.4% rms, diff-norm ~10 over 2560 elems) from the device
+  attention/norm chain; (3) the router gate weight is BYTE-IDENTICAL across
+  arms (hash e6512148 at L0) and BOTH arms' router GEMMs match host
+  reference logits computed from their own inputs (CPU 1.4e-5, TT 0.15
+  max) — the GEMMs are correct; (4) the amplification is the ROUTER GAIN:
+  the kolibri router row has ||w||_2 ~ 1 over K=2560 against logits of
+  scale O(5), so a 0.4%-rms input drift injects O(1-5) ABSOLUTE logit noise
+  (measured max 4.8-7.0 per layer, rel_rms 0.3-1.4) and the top-6-of-384
+  sigmoid-logit-add selection — a near-tie boundary — flips 2-4 of the 6
+  experts AT EVERY LAYER INCLUDING L0. The ~3% L1 moe divergence IS the
+  flipped mixture. NO local defect: staging byte-exact (pool readback,
+  prior record), dequants byte-exact, weights identical, GEMMs verified
+  against host references. ONE CONTRACT DEFECT found and fixed by the
+  instrument (kept, red-first bf16grid evidence): the TT kMatmul/kMatmulBT
+  default produced a BF16 tensor upcast on commit for f32-out callers —
+  all 384 router logits sat on the bf16 grid (bf16grid=384/384) with bf16
+  partial sums, violating the CPU row's LinearBTRaw f32 contract
+  (tenstorrent_ops.cpp: f32 out now requests dtype FLOAT32 +
+  fp32_dest_acc_en; post-fix bf16grid=0/384). GATE (ON arm, clean reset,
+  this branch, post-fix): 4 HARD flips at the SAME positions as pre-fix
+  ('der Mond...' step 7 gap -5.92 -> -5.62; 'Australia' step 5 -4.56 ->
+  -3.85; 'Wissen ist Macht' step 8 -13.35 -> -14.76; 'x1 = 3...' step 22
+  -3.06 -> -3.75), then the same deterministic memoized-expert DRAM
+  exhaustion abort — the contract fix does NOT clear the flips because the
+  driver is the upstream drift class. CLASSIFICATION: selection flips
+  DRIVEN by continuous bf16 device-arm precision drift through the
+  attention/norm chain; per the row's adjudication rule this is an
+  upstream adjudication decision (f32-enabling the device attention/norm
+  chain or accepting the band), not a local patch. ISSUE STAYS OPEN on the
+  device gates and the OOM debt.

src/vllm/model_executor/models/kolibri1_forward.cpp

@@ -36,6 +36,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <map>
+#include <set>
 #include <memory>
 #include <string>
 #include <vector>
@@ -49,6 +50,13 @@
 #include "vllm/model_executor/models/host_parallel.h"  // the ONE pool (#1664)
 #include "vt/ops.h"
 
+// SCRATCH DEBUG (layer-1 moe localization): the CPU arm's last router logits,
+// read by the TT arm's RouteDump for the direct per-expert diff.
+std::vector<float> g_kolibri1_dbg_route_logits;
+int64_t g_kolibri1_dbg_route_layer = -1;
+std::vector<uint16_t> g_kolibri1_dbg_dhn;
+int64_t g_kolibri1_dbg_dhn_layer = -1;
+
 namespace vllm {
 
 using dense_attn::DBuf;
@@ -307,14 +315,22 @@ DBuf AttentionBlock(Dev d, const Kolibri1AttnWeights& w, const Kolibri1Params& p
     };
     const int64_t n = t * hq * dh;
     for (int64_t i = 0; i < n; ++i) s += val(bf[i]);
-    std::fprintf(stderr, "[STAGE] pa_out sum=%.6f first=%.6f\n", s, val(bf[0]));
+    std::fprintf(stderr, "[STAGE][CPU] pa_out sum=%.6f first=%.6f\n", s, val(bf[0]));
   }
   return LinearBT(d, o_in, w.o_proj, t);  // [T, H]
 }
 
 // ── MoE block (kolibri1.py:126-207) ─────────────────────────────────────────
+
+// SCRATCH DEBUG (layer-1 moe localization, defined after StageDump below;
+// forward-declared here because MoeBlock calls it):
+void StageDump(const char* tag, int64_t layer, DBuf& buf, Dev d);
+void RouteDump(int64_t layer, const std::vector<float>& logits,
+               const Kolibri1HostRouting& route, int64_t t, int64_t e,
+               int64_t top_k);
+
 DBuf MoeBlock(Dev d, const Kolibri1MoeWeights& w, const Kolibri1Params& p,
-              const Tensor& dhn, int64_t t) {
+              const Tensor& dhn, int64_t t, int64_t layer) {
   const int64_t h = p.hidden_size;
   const int64_t e = p.num_experts;
   const int64_t top_k = p.num_experts_per_tok;
@@ -336,11 +352,68 @@ DBuf MoeBlock(Dev d, const Kolibri1MoeWeights& w, const Kolibri1Params& p,
     be.Synchronize(d.q);
   }
 
+  if (std::getenv("VT_KOLIBRI1_TT_STAGE_DUMP") != nullptr) {
+    static std::set<int64_t> seen;
+    if (!seen.count(layer)) {
+      seen.insert(layer);
+      Tensor bt3 = ResidentWeight(d, w.router_gate, {e, h});
+      const auto* bf3 = reinterpret_cast<const uint16_t*>(bt3.data);
+      uint32_t hh = 2166136261u;
+      double ss = 0;
+      for (int64_t i = 0; i < e * h; ++i) {
+        hh = (hh ^ bf3[i]) * 16777619u;
+        uint32_t u = static_cast<uint32_t>(bf3[i]) << 16;
+        float f;
+        std::memcpy(&f, &u, 4);
+        ss += f;
+      }
+      std::fprintf(stderr, "[STAGE][CPU] router_gate hash=%08x sum=%.4f\n",
+                   hh, ss);
+      // Host reference logits from the same dhn + router_gate bytes.
+      const int64_t hk2 = w.router_gate.shape[1];
+      std::vector<uint8_t> xb(static_cast<size_t>(t) * hk2 * 2);
+      std::memcpy(xb.data(), dhn.data, xb.size());
+      const auto* xb16 = reinterpret_cast<const uint16_t*>(xb.data());
+      std::vector<float> href(static_cast<size_t>(t) * e, 0.f);
+      for (int64_t n0 = 0; n0 < e; ++n0) {
+        const auto* wr =
+            reinterpret_cast<const uint16_t*>(bt3.data) +
+            static_cast<size_t>(n0) * hk2;
+        for (int64_t i = 0; i < t; ++i) {
+          double acc = 0;
+          for (int64_t k = 0; k < hk2; ++k) {
+            auto dec = [](uint16_t b) {
+              uint32_t u = static_cast<uint32_t>(b) << 16;
+              float f;
+              std::memcpy(&f, &u, 4);
+              return static_cast<double>(f);
+            };
+            acc += dec(wr[k]) * dec(xb16[static_cast<size_t>(i) * hk2 + k]);
+          }
+          href[static_cast<size_t>(i) * e + n0] = static_cast<float>(acc);
+        }
+      }
+      double madd = 0, mlog = 0;
+      for (size_t k = 0; k < href.size(); ++k) {
+        madd = std::max(madd, std::fabs(static_cast<double>(href[k]) -
+                                        static_cast<double>(logits[k])));
+        mlog = std::max(
+            mlog,
+            std::fabs(static_cast<double>(href[k]) -
+                      static_cast<double>(g_kolibri1_dbg_route_logits.empty()
+                                              ? 0.f
+                                              : href[k])));
+      }
+      std::fprintf(stderr, "[STAGE][CPU] router_href maxdiff_dev=%.6f\n",
+                   madd);
+    }
+  }
   Kolibri1HostRouting route;
   {
     prof::Scope prof("moe_topk");
     route = SigmoidLogitAddRouting(logits, bias, t, e, top_k);
   }
+  RouteDump(layer, logits, route, t, e, top_k);
 
   DBuf dtw(d, DType::kF32, {t, top_k},
            const_cast<float*>(route.weights.data()));
@@ -349,6 +422,7 @@ DBuf MoeBlock(Dev d, const Kolibri1MoeWeights& w, const Kolibri1Params& p,
   // Shared expert: UNGATED, always added (:146-188).
   DBuf shared = ExpertMlp(d, w.shared_experts, dhn, t,
                           p.shared_expert_intermediate_size);
+  StageDump("moe_shared", layer, shared, d);
 
   // Routed experts: the CPU-reference per-expert gather/scatter loop (the
   // mimo-v2 MoeBlock pattern).
@@ -381,6 +455,13 @@ DBuf MoeBlock(Dev d, const Kolibri1MoeWeights& w, const Kolibri1Params& p,
     }
     DBuf o = ExpertMlp(d, w.experts[static_cast<size_t>(ex)], gathered.t(), ne,
                        inter);
+    {
+      char tag[32];
+      std::snprintf(tag, sizeof tag, "moe_gath#%d", static_cast<int>(ex));
+      StageDump(tag, layer, gathered, d);
+      std::snprintf(tag, sizeof tag, "moe_ex#%d", static_cast<int>(ex));
+      StageDump(tag, layer, o, d);
+    }
     for (int64_t i = 0; i < ne; ++i) {
       d.b.Copy(d.q,
                static_cast<char*>(expert_out.ptr()) +
@@ -417,8 +498,9 @@ void StageDump(const char* tag, int64_t layer, DBuf& buf, Dev d) {
   std::vector<uint8_t> tmp(static_cast<size_t>(n) * 2);
   buf.Download(d, tmp.data());
   const auto* bf = reinterpret_cast<const uint16_t*>(tmp.data());
-  double sum = 0;
+  double sum = 0, sq = 0;
   float mx = 0;
+  uint32_t hash = 2166136261u;
   auto val = [](uint16_t bits) {
     uint32_t u = static_cast<uint32_t>(bits) << 16;
     float f;
@@ -428,12 +510,46 @@ void StageDump(const char* tag, int64_t layer, DBuf& buf, Dev d) {
   for (int64_t i = 0; i < n; ++i) {
     const float v = val(bf[i]);
     sum += v;
+    sq += static_cast<double>(v) * v;
     if (std::fabs(v) > mx) mx = std::fabs(v);
+    hash = (hash ^ bf[i]) * 16777619u;
   }
-  std::fprintf(stderr, "[STAGE] L%lld %s sum=%.6f max=%.6f first=[%.6f %.6f %.6f %.6f]\n",
-               static_cast<long long>(layer), tag, sum, mx, val(bf[0]), val(bf[1]),
+  std::fprintf(stderr, "[STAGE][CPU] L%lld %s sum=%.6f sq=%.6e hash=%08x max=%.6f first=[%.6f %.6f %.6f %.6f]\n",
+               static_cast<long long>(layer), tag, sum, sq, hash, mx, val(bf[0]), val(bf[1]),
                val(bf[2]), val(bf[3]));
 }
+// SCRATCH DEBUG (layer-1 moe localization, same env as StageDump): the
+// router's raw logits, top-k selection and weights per token — the
+// discrete-selection leg of the OFF-vs-CPU stage diff.
+void RouteDump(int64_t layer, const std::vector<float>& logits,
+               const Kolibri1HostRouting& route, int64_t t, int64_t e,
+               int64_t top_k) {
+  if (std::getenv("VT_KOLIBRI1_TT_STAGE_DUMP") == nullptr) return;
+  for (int64_t i = 0; i < t; ++i) {
+    double ls = 0;
+    for (int64_t j = 0; j < e; ++j)
+      ls += logits[static_cast<size_t>(i * e + j)];
+    std::string ids, ws;
+    for (int64_t k = 0; k < top_k; ++k) {
+      const size_t idx = static_cast<size_t>(i * top_k + k);
+      ids += (k ? "," : "") + std::to_string(route.ids[idx]);
+      ws += (k ? "," : "") +
+            [](float v) {
+              char b[32];
+              std::snprintf(b, sizeof b, "%.6f", v);
+              return std::string(b);
+            }(route.weights[idx]);
+    }
+    std::fprintf(stderr,
+                 "[STAGE][CPU] L%lld route tok=%lld logit_sum=%.6f ids=[%s] "
+                 "w=[%s]\n",
+                 static_cast<long long>(layer), static_cast<long long>(i), ls,
+                 ids.c_str(), ws.c_str());
+  }
+  g_kolibri1_dbg_route_logits = logits;
+  g_kolibri1_dbg_route_layer = layer;
+}
+
 
 }  // namespace
 
@@ -515,7 +631,7 @@ ForwardLogits ForwardKolibri1Forward(
         }
       }
       for (int64_t i = 0; i < h; ++i) s += val(bf[static_cast<size_t>(i)]);
-      std::fprintf(stderr, "[STAGE] gam sum=%.6f first=[%.6f %.6f %.6f %.6f]\n",
+      std::fprintf(stderr, "[STAGE][CPU] gam sum=%.6f first=[%.6f %.6f %.6f %.6f]\n",
                    s, val(bf[0]), val(bf[1]), val(bf[2]), val(bf[3]));
     }
     Tensor dhn_t = dhn.t();
@@ -532,6 +648,13 @@ ForwardLogits ForwardKolibri1Forward(
 
     // Attention -> post_attn_norm (no residual).
     StageDump("dhn", l, dhn, d);
+    if (l == 0 && std::getenv("VT_KOLIBRI1_TT_STAGE_DUMP") != nullptr) {
+      const int64_t hn = dhn.t().shape[0] * dhn.t().shape[1];
+      g_kolibri1_dbg_dhn.assign(
+          reinterpret_cast<const uint16_t*>(dhn.t().data),
+          reinterpret_cast<const uint16_t*>(dhn.t().data) + hn);
+      g_kolibri1_dbg_dhn_layer = l;
+    }
     DBuf attn = AttentionBlock(d, lw.attn, p, lw.is_sliding, dhn.t(),
                                si.positions.t(), si, *kv_ptr, t);
     StageDump("attn", l, attn, d);
@@ -561,7 +684,7 @@ ForwardLogits ForwardKolibri1Forward(
     StageDump("dh2", l, dh2, d);
 
     // MoE on EVERY layer -> post_ffn_norm (no residual).
-    DBuf moe = MoeBlock(d, lw.moe, p, dh2.t(), t);
+    DBuf moe = MoeBlock(d, lw.moe, p, dh2.t(), t, l);
     StageDump("moe", l, moe, d);
     DBuf moe_n(d, DType::kBF16, {t, h});
     Tensor w_pf = ResidentWeight(d, lw.post_ffn_norm, {h});