Repository navigation
Expand file tree
/
Copy pathindex.html
More file actions
1916 lines (1882 loc) · 443 KB
/
Copy pathindex.html
File metadata and controls
1916 lines (1882 loc) · 443 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!doctype html>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Model Flow</title>
<meta name="claude:viewport-fit-sides" content="cover">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Fraunces:opsz,wght@9..144,500;9..144,650&family=Public+Sans:wght@400;600&family=JetBrains+Mono:wght@400;500&display=swap">
<style>
/* Layout: one reading column (68ch) with full-width figures; each claim = heading, short text, figure, caption. */
:root {
--bg: #f3f5f4; --panel: #ffffff; --ink: #18211f; --muted: #5b6763; --rule: #d9dfdc; --accent: #1f5fbf; --accent-soft: #dbe6f7;
--warm: #c2582b; --good: #2f7d4f; --good-soft: #dcefe3; --bad: #a33a3a; --bad-soft: #f5e0de; --note: #eef3f1;
--c0: #3b6fb6; --c1: #c2582b; --c2: #3f8f5a; --c3: #8a5bb0; --c4: #b8912a; --c5: #2d8c93; --c6: #b5476f; --c7: #6d7a3a; --c8: #7a6553;
--display: "Fraunces", Georgia, serif; --body: "Public Sans", system-ui, sans-serif; --mono: "JetBrains Mono", ui-monospace, monospace;
}
@media (prefers-color-scheme: dark) { :root:not([data-theme="light"]) {
--bg: #111615; --panel: #182020; --ink: #e4ebe8; --muted: #93a19c; --rule: #2b3634; --accent: #7eaaf0; --accent-soft: #1d2b40;
--warm: #e58a5f; --good: #74c08c; --good-soft: #1c3326; --bad: #e08080; --bad-soft: #3a2224; --note: #1a2322;
--c0: #7ba4e6; --c1: #e58a5f; --c2: #74c08c; --c3: #b793dc; --c4: #dcbd5e; --c5: #62c0c6; --c6: #e3809f; --c7: #a8b86a; --c8: #b49c88;
color-scheme: dark; } }
:root[data-theme="dark"] {
--bg: #111615; --panel: #182020; --ink: #e4ebe8; --muted: #93a19c; --rule: #2b3634; --accent: #7eaaf0; --accent-soft: #1d2b40;
--warm: #e58a5f; --good: #74c08c; --good-soft: #1c3326; --bad: #e08080; --bad-soft: #3a2224; --note: #1a2322;
--c0: #7ba4e6; --c1: #e58a5f; --c2: #74c08c; --c3: #b793dc; --c4: #dcbd5e; --c5: #62c0c6; --c6: #e3809f; --c7: #a8b86a; --c8: #b49c88;
color-scheme: dark; }
body { background: var(--bg); color: var(--ink); font: 16.5px/1.65 var(--body); }
.wrap { max-width: 1100px; margin: 0 auto; padding-left: max(16px, env(safe-area-inset-left, 0px)); padding-right: max(16px, env(safe-area-inset-right, 0px)); padding-block: 40px 80px; display: grid; gap: 34px; }
.prose { max-width: 68ch; display: grid; gap: 14px; }
.prose p, .prose ul { margin: 0; }
.prose ul { padding-left: 1.2em; display: grid; gap: 6px; }
h1 { font: 650 clamp(32px, 5.4vw, 50px)/1.05 var(--display); margin: 0; letter-spacing: -0.015em; text-wrap: balance; }
h2 { font: 600 clamp(22px, 3vw, 29px)/1.2 var(--display); margin: 8px 0 0; text-wrap: balance; }
.lede { font-size: 19px; line-height: 1.55; color: var(--muted); }
.eyebrow { font: 500 11px/1.4 var(--mono); text-transform: uppercase; letter-spacing: 0.12em; color: var(--accent); }
a { color: var(--accent); }
.num { font-variant-numeric: tabular-nums; }
.claims { list-style: none; padding: 0; margin: 0; display: grid; gap: 10px; counter-reset: c; }
.claims li { display: grid; grid-template-columns: 2.2em minmax(0, 1fr) auto; gap: 4px 14px; align-items: baseline;
padding: 14px 16px; background: var(--panel); border: 1px solid var(--rule); border-radius: 6px; counter-increment: c; }
.claims li::before { content: counter(c); font: 600 22px/1 var(--display); color: var(--accent); grid-row: span 2; }
.claims b { font-weight: 600; }
.claims .ev { grid-column: 2; font-size: 14.5px; color: var(--muted); }
.claims a { grid-column: 3; grid-row: 1; font: 500 12px var(--mono); text-decoration: none; white-space: nowrap; }
@media (max-width: 640px) { .claims li { grid-template-columns: 2em minmax(0, 1fr); } .claims a { grid-column: 2; grid-row: auto; } }
.chip { display: inline-block; font: 500 10.5px/1 var(--mono); letter-spacing: 0.06em; text-transform: uppercase; padding: 4px 6px;
border-radius: 3px; vertical-align: 2px; margin-left: 6px; white-space: nowrap; }
.chip.pre { background: var(--good-soft); color: var(--good); }
.chip.exp { background: var(--accent-soft); color: var(--accent); }
.chip.fail { background: var(--bad-soft); color: var(--bad); }
figure { margin: 0; background: var(--panel); border: 1px solid var(--rule); border-radius: 6px; padding: 18px; display: grid; gap: 12px; min-width: 0; }
figcaption { font-size: 14px; color: var(--muted); max-width: 92ch; line-height: 1.55; }
figcaption b { color: var(--ink); font-weight: 600; }
.scroll { overflow-x: auto; }
svg { display: block; width: 100%; height: auto; }
svg text { fill: var(--muted); font: 11px var(--mono); }
svg .ink { fill: var(--ink); }
svg .warn { fill: var(--bad); }
table.methods { border-collapse: collapse; font-size: 14px; line-height: 1.45; min-width: 640px; }
table.methods th { text-align: left; font: 600 12px/1.3 var(--mono); color: var(--muted); text-transform: uppercase; letter-spacing: 0.05em; padding: 8px 10px; border-bottom: 2px solid var(--rule); }
table.methods td { padding: 8px 10px; border-bottom: 1px solid var(--rule); vertical-align: top; }
table.methods td:first-child { font-weight: 600; }
table.methods td:last-child { color: var(--muted); }
a.cite { text-decoration: none; border-bottom: 1px dotted var(--accent); }
.prior-wrap { max-width: 1100px; }
table.prior td:nth-child(2) { color: var(--ink); }
table.prior td:last-child { white-space: nowrap; }
ol.refs { font-size: 14px; line-height: 1.5; padding-left: 22px; display: grid; gap: 6px; }
ol.refs li:target { background: var(--accent-soft); }
.tells { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); gap: 12px; }
.tells > div { background: var(--panel); border: 1px solid var(--rule); border-radius: 6px; padding: 12px 16px; font-size: 15px; }
.tells > div:first-child { border-top: 3px solid var(--good); }
.tells > div:last-child { border-top: 3px solid var(--warm); }
.tells ul { margin: 6px 0 0; padding-left: 18px; }
.tells li { margin: 4px 0; }
.ktoggle { display: flex; gap: 6px; }
.ktoggle button { font: 500 11.5px/1 var(--mono); padding: 6px 10px; border: 1px solid var(--rule); border-radius: 4px;
background: var(--panel); color: var(--muted); cursor: pointer; }
.ktoggle button[aria-pressed="true"] { background: var(--accent-soft); color: var(--accent); border-color: var(--accent); }
.controls { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; }
label.ctl { max-width: 100%; min-width: 0; flex-wrap: wrap; font: 500 11px/1 var(--mono); color: var(--muted); text-transform: uppercase; letter-spacing: 0.08em; display: inline-flex; gap: 8px; align-items: center; }
select { max-width: 100%; min-width: 0; font: 13px var(--body); padding: 6px 8px; border: 1px solid var(--rule); border-radius: 4px; background: var(--panel); color: var(--ink); }
select:focus-visible, a:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; }
.legend { display: flex; flex-wrap: wrap; gap: 6px 16px; font-size: 13px; color: var(--muted); }
.legend i { display: inline-block; width: 12px; height: 12px; border-radius: 2px; margin-right: 6px; vertical-align: -1px; }
.legend i.line { height: 3px; vertical-align: 3px; border-radius: 0; }
.note { background: var(--note); border-radius: 6px; padding: 14px 16px; font-size: 15px; line-height: 1.6; }
.fails { display: grid; gap: 8px; }
.fails div { display: grid; grid-template-columns: minmax(0, 1fr); gap: 2px; padding: 10px 14px; border-left: 3px solid var(--bad); background: var(--panel); }
.fails b { font-weight: 600; }
.fails span { font-size: 14.5px; color: var(--muted); }
hr { border: 0; border-top: 1px solid var(--rule); margin: 0; }
h3.tsub { font: 600 18px/1.3 var(--display); margin: 8px 0 0; }
.small { font-size: 14px; color: var(--muted); }
.excols{display:grid;grid-template-columns:1fr 1fr;gap:12px}.excols h4{margin:0 0 4px;font-size:13px;color:var(--muted);font-weight:600}.excols ol{margin:0;padding-left:22px;font-size:13px;line-height:1.5}
</style>
<main class="wrap">
<header class="prose">
<span class="eyebrow">Model Flow · October 2026 · version 0.3</span>
<h1>Model Flow</h1>
<p class="lede">What does a language model look like if you study it the way you would study a fluid: not
molecule by molecule, but by how the whole body of it moves? Here, the molecules are the model's weights and
circuits, and the fluid is thousands of texts flowing through the model's layers.</p>
<p>This project does not try to explain individual weights. It follows a population of texts through a model and
measures where the population holds its shape and where it reorganizes. Three findings:</p>
<ul>
<li><b>Most of the flow is smooth.</b> Stretches of layers share coordinates, partly because a running total
resists change even in an untrained model.</li>
<li><b>Some apparent sharp changes are a few huge coordinates.</b> In Qwen3 and SmolLM2 an apparent break
becomes an ordinary layer once each coordinate is rescaled by its typical spread.</li>
<li><b>Gemma 4 has one real turn.</b> Two layers write far more than the running total holds, and a stored
scalar shrinks what was there, so the axes change completely while much of each text's neighbourhood survives.
The stored scalars alone predicted where a larger Gemma 4 turns. The model's predictions tolerate the turn
because later layers reuse what the turn layers computed for attention.</li>
</ul>
<div class="note"><b>How this was made.</b> This is a human–AI collaboration. The author set the question and the
ideas behind it, chose the directions, and reviewed each step. Claude (Anthropic), working in Claude Code, wrote
the code, ran the experiments, searched the literature and drafted this page, including proposing the checks that
narrowed several claims. Results and wording were reviewed together.</div>
</header>
<section class="prose" id="s-residual" aria-labelledby="h-residual">
<h2 id="h-residual">One running total</h2>
<p>A language model reads text as tokens (words or pieces of words) and turns each token into a vector, a list of a
few thousand numbers. The vectors then pass through a stack of <b>layers</b>. Each layer reads the current vector,
computes something nonlinear from it, and <em>adds</em> its result, its <b>write</b>, back on. The running total
is called the <b>residual stream</b> (<a class="cite" href="#r-elhage">Elhage et al. 2021</a>).</p>
<p>Each layer writes in two steps. <b>Attention</b> lets each token read from the tokens before it, through several
parallel readers called <b>heads</b>. The <b>MLP</b>, a small two-stage network, then transforms each token's
vector on its own. Both steps add their results to the residual stream.</p>
<p>Because every layer adds to the same running total, the vector after layer 9 and the vector after layer 18 live
in the same space, with the same axes. That is what makes the fluid picture possible: one space, and a flow through
it, layer by layer.</p>
</section>
<section class="prose" id="s-cloud" aria-labelledby="h-cloud">
<h2 id="h-cloud">A cloud of texts</h2>
<p>One text gives one point per layer. We send thousands of short encyclopedia articles (DBpedia summaries, at most
64 tokens) through an already-trained model, without changing its weights (a <b>frozen</b> model), and represent
each article at each layer by the average of its token vectors. That gives a cloud of points at every layer. The
question is how the cloud changes from one layer to the next. Where something is fitted to articles (a grouping, a
map between layers), it is tested on <b>held-out</b> articles that the fit never saw.</p>
<p>We compare two layers in two ways:</p>
<ul>
<li><b>Place.</b> Does an article keep its coordinates? We take the cosine between its vectors at the two layers,
after subtracting each layer's average vector: the <b>centered cosine</b>. 1 means the same direction, 0 means
unrelated.</li>
<li><b>Company.</b> Does an article keep its neighbours? We list its 10 nearest articles at each layer and count
how many are shared. This ignores the axes entirely: if the whole cloud rotated, every article would keep all its
neighbours.</li>
</ul>
<p class="small">Models: Gemma 4 E2B and E4B, Qwen3-1.7B, Qwen3.5-2B, SmolLM2-1.7B, Pythia-410M (untrained and
trained), and one encoder, ModernBERT-large. All public, all frozen.</p>
</section>
<section class="prose" id="s-see" aria-labelledby="h-see">
<h2 id="h-see">What we see</h2>
<p>Comparing every pair of layers gives a map of the whole model. In Gemma 4 E2B the map has two features. Dark square
blocks are stretches of layers that share coordinates. And at layer 14 there is a sharp break: across that one
layer, articles' coordinates become unrelated. We call a layer like that a <b>turn</b>.</p>
</section>
<figure>
<div class="controls"><label class="ctl" for="heatSet">model <select id="heatSet"></select></label></div>
<div class="scroll"><svg id="heat" role="img" aria-label="Layer by layer heatmaps of same place and same company"></svg></div>
<figcaption><b>Every pair of layers, compared.</b> Left: same place, as a share: of 3,000 held-out articles, how many
have their own vector as the nearest point at the other layer (after centering). Right: share of 10 neighbours kept (same company). Darker is more similar.
Switch models to compare; hover a cell for its value.</figcaption>
</figure>
<section class="prose">
<p>Other models show breaks too, if measured plainly: Qwen3 seems to turn at layer 2 and SmolLM2 at layer 7.
Before reading anything into these, each needs a reference.</p>
</section>
<section class="prose" id="s-real" aria-labelledby="h-real">
<h2 id="h-real">Is it real? Three controls</h2>
<h3 class="tsub">An untrained model: staying put is the default</h3>
<p>In an untrained Pythia (choose "pythia_0" above), every article still finds itself at any layer up to six layers
away, and keeps most of its neighbours at that distance. Each layer adds a small random write to a large running
total, so nothing moves far quickly. So blocks of shared coordinates are not, by themselves, something the model
learned (<a class="cite" href="#r-veit">Veit et al. 2016</a>). Training shows up as departures from this.</p>
<h3 class="tsub">Robust scaling: a change in a few huge coordinates can look like a turn of the whole cloud</h3>
<p>Language models carry a handful of coordinates with values hundreds of times larger than the rest
(<a class="cite" href="#r-kovaleva">Kovaleva et al. 2021</a>; <a class="cite" href="#r-massive">Sun et al. 2024</a>).
When one switches on, it dominates every cosine. Rescale each coordinate by its typical spread, and the Qwen3 and
SmolLM2 breaks become ordinary layers. Qwen3, Qwen3.5 and SmolLM2 then keep their coordinates all the way
through. Gemma's turn stays.</p>
<p>The huge coordinates are not noise. In Qwen3, three coordinates switch on at layer 2, almost entirely on the first
token of each text, which becomes the token every other token can attend to by default (an attention sink,
<a class="cite" href="#r-massive">Sun et al. 2024</a>). Blocking that switch breaks the model. But the other
2,045 coordinates keep their arrangement across layer 2. So the event is real and needed, and it is a change in a
few coordinates of one token, not a turn of the whole cloud like Gemma's.</p>
<h3 class="tsub">A second model as reference: rotations link layers, and they link models</h3>
<p>To ask whether information survives a layer, fit a <b>rotation</b> from one layer's cloud to another's on some
articles, then check whether it carries new articles to themselves. It does, very well. The same procedure works
almost as well between two <em>different</em> models.</p>
</section>
<figure>
<div class="scroll"><svg id="rot" role="img" aria-label="How often a fitted rotation finds the same article again, within and across models"></svg></div>
<div class="legend"><span><i style="background:var(--muted)"></i>no map</span><span><i style="background:var(--accent)"></i>within one model</span><span><i style="background:var(--c2)"></i>across different models</span><span><i style="background:var(--bad)"></i>shuffled pairs (null)</span></div>
<figcaption><b>Share of 3,000 held-out articles whose nearest neighbour, after the fitted map, is the same
article</b> (chance 1 in 3,000; maps fit on 10,000 other articles). Fitting on shuffled pairs falls to chance.</figcaption>
</figure>
<section class="prose">
<p>Both results point the same way: the arrangement of articles is shared, between layers of one model and between
different models (<a class="cite" href="#r-huh">Huh et al. 2024</a>), and a rotation carries one onto the other.</p>
<p>The second model also sets a limit on what a good fit means. Two different models were never rotations of each
other, yet one fits onto the other almost as well as one layer onto the next. So a good fit between two layers does
not show that one layer rotated the other. It shows only that both arrange the articles in a similar way
(<a class="cite" href="#r-ding21">Ding et al. 2021</a>). And the rotation carries the arrangement, not everything:
about half of each vector's length is left unexplained.</p>
</section>
<section class="prose" id="s-turn" aria-labelledby="h-turn">
<h2 id="h-turn">Gemma 4's turn</h2>
<p>One break survives every control. This section describes it, explains how it happens, tests the explanation on
a second model, and asks why the model's predictions tolerate it.</p>
<h3 class="tsub">What it looks like</h3>
<p>Across layers 13 to 15 of Gemma 4 E2B, articles' coordinates become unrelated, yet each article keeps about half
of its 10 neighbours, where neighbouring layers elsewhere keep about four in five. The turn holds on a second
sample, at 1,000 to 6,000 articles, with the huge coordinates removed, in 32-bit and 16-bit arithmetic, and for
single tokens as well as whole articles. Here is one article, chosen by a fixed rule (the first in the sample with
the median number of neighbours kept):</p>
</section>
<figure>
<div id="example"></div>
<figcaption><b>One article across the turn.</b> Its 10 nearest neighbours at layer 13 and at layer 15; shared ones in
bold. Its own centered cosine across the turn is <span id="exCos"></span>: its coordinates are unrelated. Yet half
its company is the same, and the rest is the same kind of thing.</figcaption>
</figure>
<section class="prose">
<p>To see the turn as a flow, we need one more idea. At each layer we sort the articles into a few <b>groups</b> with
k-means, a standard method that splits points into a chosen number of groups by distance, using no labels. k-means
starts from random points, so we run it twice and score how well the two runs agree: the <b>repeatability</b>
(1 = identical, 0 = chance). Then we follow the same articles from one layer's groups to the next.</p>
</section>
<figure>
<div class="controls"><div class="ktoggle" id="flowK" role="group" aria-label="Groups per layer"></div>
<div class="ktoggle" id="flowCols" role="group" aria-label="Layers shown"></div></div>
<div class="scroll"><svg id="flow" role="img" aria-label="Flow of 1,500 articles through groups across layers of Gemma 4 E2B"></svg></div>
<div class="legend" id="flowLegend"></div>
<div class="scroll"><svg id="flowStats" role="img" aria-label="Per layer: topic agreement of the groups and how much of the grouping carries to the next layer"></svg></div>
<div class="legend"><span><i class="line" style="background:var(--accent)"></i>topic agreement of the groups</span><span><i class="line" style="background:var(--warm)"></i>grouping carried to the next layer</span><span><i class="line" style="background:var(--muted)"></i>repeatability</span></div>
<figcaption><b>Gemma 4 E2B, 1,500 held-out articles.</b> Bands follow the same articles between layers. Colours show
the topic most over-represented in each group, added after the grouping. Repeatability under each layer (median
<span id="flowRep4"></span> with four groups, <span id="flowRep8"></span> with eight); layers below 0.8 are faded.
Lower panel: topic agreement (adjusted mutual information with the nine DBpedia topics) and carry-over (adjusted
Rand index between consecutive layers). Try "layers 10–20". Hover a band for its size.</figcaption>
</figure>
<section class="prose">
<p>Most of the depth is steady flow: groups carry over from layer to layer. Around the turn the carry-over drops and
topic agreement roughly halves (lower panel), then the groups re-form a few layers later. The map shows who moves
with whom, not why. Its groups are our choice of resolution, not the model's concepts.</p>
</section>
<section class="prose" id="s-how" aria-labelledby="h-how">
<h3 class="tsub" id="h-how">How it happens</h3>
<p>After each layer, Gemma 4 multiplies the whole residual stream by a fixed number stored in the checkpoint, the
<b>layer scalar</b> (<a class="cite" href="#r-gemma4src">layer_scalar</a>). Gemma's technical report says it is
there to keep values within 16-bit range (<a class="cite" href="#r-gemma4report">Gemma Team 2026</a>). Unlike
similar tricks that scale only a layer's write (<a class="cite" href="#r-layerscale">Touvron et al. 2021</a>), it
scales the running total too.</p>
<ul>
<li><b>Two layers out-write the stream.</b> Layers 13 and 14 write vectors many times the size of the residual, at
right angles to it. Their small scalars then shrink the sum. The old residual becomes a small remainder next to
the new write, so the coordinates turn. Nothing can be deleted from a running total; writing something much
larger is the way to overwrite it, an idea proposed for GPT-2 by
<a class="cite" href="#r-heimersheim">Heimersheim & Turner 2023</a>.</li>
<li><b>The scalar sets how sharp the turn is.</b> Multiply the scalars of layers 13–14 by a factor and the turn
changes smoothly: smaller scalars give a sharper turn, and with both scalars set to 1 the turn across layer 14
is about that of an ordinary layer.</li>
<li><b>The MLPs make most of the write.</b> Replacing the MLP output of layer 13 or 14 with its average moves the
turn to the next layers rather than removing it. Removing single attention heads barely matters.</li>
</ul>
</section>
<figure>
<div class="scroll"><svg id="tracks" role="img" aria-label="Gemma 4 E2B per-layer tracks: blocks, coordinate turn, layer write, scalar, attention"></svg></div>
<figcaption><b>Gemma 4 E2B, one column per layer.</b> Blocks: stretches of layers that share coordinates. Coordinate
turn: 1 − centered cosine between a layer's input and output. Layer write: the layer's own write relative to the
residual, scalar divided out (log scale). Layer scalar: the stored multiplier. Attention: full-attention layers
(circles; size = strongest induction head) and mean attention to the first token. Shaded: layers that reuse earlier
layers' attention keys and values.</figcaption>
</figure>
<section class="prose">
<p><b>Size sets the turn; content sets the company.</b> A cosine only sees whether the axes changed, not what was
written. So for single tokens we also track company: which 10 other tokens are closest after layer 14, and whether
they are the same ones as in the unchanged model. We then changed the writes of layers 13 and 14 in six ways, from
setting 1,024 random MLP units to their average value up to replacing the whole write with noise of the same
size.</p>
</section>
<figure>
<div class="scroll"><svg xmlns:xlink="http://www.w3.org/1999/xlink" width="100%" style="max-width:900px;height:auto;display:block" role="img" aria-label="Gemma 4 E2B: changes at layers 13-14 and their effect on the turn, company and predictions; layer 12 scalar dose-response" viewBox="0 0 685.7175 279.106329" xmlns="http://www.w3.org/2000/svg" version="1.1">
<metadata>
<rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<cc:Work>
<dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
<dc:date>2026-10-19T10:01:50.030141</dc:date>
<dc:format>image/svg+xml</dc:format>
<dc:creator>
<cc:Agent>
<dc:title>Matplotlib v3.10.3, https://matplotlib.org/</dc:title>
</cc:Agent>
</dc:creator>
</cc:Work>
</rdf:RDF>
</metadata>
<defs>
<style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style>
</defs>
<g id="figure_1">
<g id="patch_1">
<path d="M -0 279.106329
L 685.7175 279.106329
L 685.7175 0
L -0 0
L -0 279.106329
z
" style="fill: none"/>
</g>
<g id="axes_1">
<g id="patch_2">
<path d="M 88.6375 216.492038
L 427.217113 216.492038
L 427.217113 20.798437
L 88.6375 20.798437
L 88.6375 216.492038
z
" style="fill: none"/>
</g>
<g id="matplotlib.axis_1">
<g id="xtick_1">
<g id="line2d_1">
<path d="M 88.6375 216.492038
L 88.6375 20.798437
" clip-path="url(#p38f4ca0da5)" style="fill: none; stroke: var(--rule); stroke-width: 0.6; stroke-linecap: square"/>
</g>
<g id="line2d_2">
<defs>
<path id="mdb9d84bdd0" d="M 0 0
L 0 3.5
" style="stroke: var(--muted); stroke-width: 0.8"/>
</defs>
<g>
<use xlink:href="#mdb9d84bdd0" x="88.6375" y="216.492038" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_1">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: middle; fill: var(--muted)" x="88.6375" y="230.330631" transform="rotate(-0 88.6375 230.330631)">0.0</text>
</g>
</g>
<g id="xtick_2">
<g id="line2d_3">
<path d="M 155.025659 216.492038
L 155.025659 20.798437
" clip-path="url(#p38f4ca0da5)" style="fill: none; stroke: var(--rule); stroke-width: 0.6; stroke-linecap: square"/>
</g>
<g id="line2d_4">
<g>
<use xlink:href="#mdb9d84bdd0" x="155.025659" y="216.492038" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_2">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: middle; fill: var(--muted)" x="155.025659" y="230.330631" transform="rotate(-0 155.025659 230.330631)">0.2</text>
</g>
</g>
<g id="xtick_3">
<g id="line2d_5">
<path d="M 221.413819 216.492038
L 221.413819 20.798437
" clip-path="url(#p38f4ca0da5)" style="fill: none; stroke: var(--rule); stroke-width: 0.6; stroke-linecap: square"/>
</g>
<g id="line2d_6">
<g>
<use xlink:href="#mdb9d84bdd0" x="221.413819" y="216.492038" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_3">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: middle; fill: var(--muted)" x="221.413819" y="230.330631" transform="rotate(-0 221.413819 230.330631)">0.4</text>
</g>
</g>
<g id="xtick_4">
<g id="line2d_7">
<path d="M 287.801978 216.492038
L 287.801978 20.798437
" clip-path="url(#p38f4ca0da5)" style="fill: none; stroke: var(--rule); stroke-width: 0.6; stroke-linecap: square"/>
</g>
<g id="line2d_8">
<g>
<use xlink:href="#mdb9d84bdd0" x="287.801978" y="216.492038" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_4">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: middle; fill: var(--muted)" x="287.801978" y="230.330631" transform="rotate(-0 287.801978 230.330631)">0.6</text>
</g>
</g>
<g id="xtick_5">
<g id="line2d_9">
<path d="M 354.190137 216.492038
L 354.190137 20.798437
" clip-path="url(#p38f4ca0da5)" style="fill: none; stroke: var(--rule); stroke-width: 0.6; stroke-linecap: square"/>
</g>
<g id="line2d_10">
<g>
<use xlink:href="#mdb9d84bdd0" x="354.190137" y="216.492038" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_5">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: middle; fill: var(--muted)" x="354.190137" y="230.330631" transform="rotate(-0 354.190137 230.330631)">0.8</text>
</g>
</g>
<g id="xtick_6">
<g id="line2d_11">
<path d="M 420.578297 216.492038
L 420.578297 20.798437
" clip-path="url(#p38f4ca0da5)" style="fill: none; stroke: var(--rule); stroke-width: 0.6; stroke-linecap: square"/>
</g>
<g id="line2d_12">
<g>
<use xlink:href="#mdb9d84bdd0" x="420.578297" y="216.492038" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_6">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: middle; fill: var(--muted)" x="420.578297" y="230.330631" transform="rotate(-0 420.578297 230.330631)">1.0</text>
</g>
</g>
<g id="text_7">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: middle; fill: var(--ink)" x="257.927306" y="243.040944" transform="rotate(-0 257.927306 243.040944)">fraction (1 = full turn / all kept)</text>
</g>
</g>
<g id="matplotlib.axis_2">
<g id="ytick_1">
<g id="line2d_13">
<defs>
<path id="m31a93be521" d="M 0 0
L -3.5 0
" style="stroke: var(--muted); stroke-width: 0.8"/>
</defs>
<g>
<use xlink:href="#m31a93be521" x="88.6375" y="41.697455" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_8">
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: end; fill: var(--muted)" x="81.6375" y="44.73683" transform="rotate(-0 81.6375 44.73683)">no change</text>
</g>
</g>
<g id="ytick_2">
<g id="line2d_14">
<g>
<use xlink:href="#m31a93be521" x="88.6375" y="72.476568" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_9">
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; fill: var(--muted)" transform="translate(25.165 71.644693)">1,024 random</text>
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; fill: var(--muted)" transform="translate(13.53 80.602943)">neurons to mean</text>
</g>
</g>
<g id="ytick_3">
<g id="line2d_15">
<g>
<use xlink:href="#m31a93be521" x="88.6375" y="103.255681" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_10">
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; fill: var(--muted)" transform="translate(28.61375 102.423806)">1,024 largest</text>
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; fill: var(--muted)" transform="translate(13.53 111.382056)">neurons to mean</text>
</g>
</g>
<g id="ytick_4">
<g id="line2d_16">
<g>
<use xlink:href="#m31a93be521" x="88.6375" y="134.034794" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_11">
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; fill: var(--muted)" transform="translate(28.355 133.202919)">every neuron</text>
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; fill: var(--muted)" transform="translate(36.3075 142.161169)">to its mean</text>
</g>
</g>
<g id="ytick_5">
<g id="line2d_17">
<g>
<use xlink:href="#m31a93be521" x="88.6375" y="164.813907" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_12">
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; fill: var(--muted)" transform="translate(7.2 163.982032)">every neuron from</text>
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; fill: var(--muted)" transform="translate(32.02 172.940282)">another text</text>
</g>
</g>
<g id="ytick_6">
<g id="line2d_18">
<g>
<use xlink:href="#m31a93be521" x="88.6375" y="195.59302" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_13">
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; fill: var(--muted)" transform="translate(24.3475 194.761145)">write replaced</text>
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; fill: var(--muted)" transform="translate(48.00375 203.719395)">by noise</text>
</g>
</g>
</g>
<g id="patch_3">
<path d="M 88.6375 37.69617
L 418.377158 37.69617
L 418.377158 29.693601
L 88.6375 29.693601
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--accent)"/>
</g>
<g id="patch_4">
<path d="M 88.6375 68.475283
L 418.233669 68.475283
L 418.233669 60.472714
L 88.6375 60.472714
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--accent)"/>
</g>
<g id="patch_5">
<path d="M 88.6375 99.254396
L 418.780312 99.254396
L 418.780312 91.251827
L 88.6375 91.251827
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--accent)"/>
</g>
<g id="patch_6">
<path d="M 88.6375 130.033509
L 410.991902 130.033509
L 410.991902 122.03094
L 88.6375 122.03094
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--accent)"/>
</g>
<g id="patch_7">
<path d="M 88.6375 160.812622
L 418.969475 160.812622
L 418.969475 152.810053
L 88.6375 152.810053
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--accent)"/>
</g>
<g id="patch_8">
<path d="M 88.6375 191.591735
L 419.527239 191.591735
L 419.527239 183.589166
L 88.6375 183.589166
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--accent)"/>
</g>
<g id="patch_9">
<path d="M 88.6375 45.69874
L 420.578297 45.69874
L 420.578297 37.69617
L 88.6375 37.69617
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--warm)"/>
</g>
<g id="patch_10">
<path d="M 88.6375 76.477853
L 349.211026 76.477853
L 349.211026 68.475283
L 88.6375 68.475283
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--warm)"/>
</g>
<g id="patch_11">
<path d="M 88.6375 107.256966
L 236.799275 107.256966
L 236.799275 99.254396
L 88.6375 99.254396
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--warm)"/>
</g>
<g id="patch_12">
<path d="M 88.6375 138.036079
L 162.726686 138.036079
L 162.726686 130.033509
L 88.6375 130.033509
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--warm)"/>
</g>
<g id="patch_13">
<path d="M 88.6375 168.815192
L 91.309623 168.815192
L 91.309623 160.812622
L 88.6375 160.812622
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--warm)"/>
</g>
<g id="patch_14">
<path d="M 88.6375 199.594305
L 91.376012 199.594305
L 91.376012 191.591735
L 88.6375 191.591735
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--warm)"/>
</g>
<g id="patch_15">
<path d="M 88.6375 53.701309
L 420.578297 53.701309
L 420.578297 45.69874
L 88.6375 45.69874
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--good)"/>
</g>
<g id="patch_16">
<path d="M 88.6375 84.480422
L 395.35689 84.480422
L 395.35689 76.477853
L 88.6375 76.477853
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--good)"/>
</g>
<g id="patch_17">
<path d="M 88.6375 115.259535
L 330.356754 115.259535
L 330.356754 107.256966
L 88.6375 107.256966
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--good)"/>
</g>
<g id="patch_18">
<path d="M 88.6375 146.038648
L 287.700415 146.038648
L 287.700415 138.036079
L 88.6375 138.036079
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--good)"/>
</g>
<g id="patch_19">
<path d="M 88.6375 176.817761
L 229.527551 176.817761
L 229.527551 168.815192
L 88.6375 168.815192
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--good)"/>
</g>
<g id="patch_20">
<path d="M 88.6375 207.596874
L 187.435449 207.596874
L 187.435449 199.594305
L 88.6375 199.594305
z
" clip-path="url(#p38f4ca0da5)" style="fill: var(--good)"/>
</g>
<g id="patch_21">
<path d="M 88.6375 216.492038
L 88.6375 20.798437
" style="fill: none; stroke: var(--muted); stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="patch_22">
<path d="M 88.6375 216.492038
L 427.217113 216.492038
" style="fill: none; stroke: var(--muted); stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="text_14">
<text style="font-weight: 600; font-size: 10px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: start" x="88.6375" y="14.798437" transform="rotate(-0 88.6375 14.798437)">Content changes, the turn does not</text>
</g>
<g id="legend_1">
<g id="patch_23">
<path d="M 100.405627 256.33797
L 115.405627 256.33797
L 115.405627 251.08797
L 100.405627 251.08797
z
" style="fill: var(--accent)"/>
</g>
<g id="text_15">
<text style="font-size: 7.5px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: start" x="121.405627" y="256.33797" transform="rotate(-0 121.405627 256.33797)">coordinate turn across layers 13-14</text>
</g>
<g id="patch_24">
<path d="M 100.405627 267.346563
L 115.405627 267.346563
L 115.405627 262.096563
L 100.405627 262.096563
z
" style="fill: var(--warm)"/>
</g>
<g id="text_16">
<text style="font-size: 7.5px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: start" x="121.405627" y="267.346563" transform="rotate(-0 121.405627 267.346563)">company kept after layer 14</text>
</g>
<g id="patch_25">
<path d="M 269.896252 256.33797
L 284.896252 256.33797
L 284.896252 251.08797
L 269.896252 251.08797
z
" style="fill: var(--good)"/>
</g>
<g id="text_17">
<text style="font-size: 7.5px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: start" x="290.896252" y="256.33797" transform="rotate(-0 290.896252 256.33797)">next-token prediction unchanged</text>
</g>
</g>
</g>
<g id="axes_2">
<g id="patch_26">
<path d="M 479.353022 216.492038
L 678.5175 216.492038
L 678.5175 20.798437
L 479.353022 20.798437
L 479.353022 216.492038
z
" style="fill: none"/>
</g>
<g id="patch_27">
<path d="M 479.353022 158.935096
L 678.5175 158.935096
L 678.5175 101.378155
L 479.353022 101.378155
z
" clip-path="url(#p7eeb96b57c)" style="fill: var(--muted); opacity: 0.12"/>
</g>
<g id="matplotlib.axis_3">
<g id="xtick_7">
<g id="line2d_19">
<g>
<use xlink:href="#mdb9d84bdd0" x="488.405953" y="216.492038" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_18">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: middle; fill: var(--muted)" x="488.405953" y="230.330631" transform="rotate(-0 488.405953 230.330631)">x0.5</text>
</g>
</g>
<g id="xtick_8">
<g id="line2d_20">
<g>
<use xlink:href="#mdb9d84bdd0" x="548.758825" y="216.492038" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_19">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: middle; fill: var(--muted)" x="548.758825" y="230.330631" transform="rotate(-0 548.758825 230.330631)">x1</text>
</g>
</g>
<g id="xtick_9">
<g id="line2d_21">
<g>
<use xlink:href="#mdb9d84bdd0" x="609.111697" y="216.492038" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_20">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: middle; fill: var(--muted)" x="609.111697" y="230.330631" transform="rotate(-0 609.111697 230.330631)">x2</text>
</g>
</g>
<g id="xtick_10">
<g id="line2d_22">
<g>
<use xlink:href="#mdb9d84bdd0" x="669.464569" y="216.492038" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_21">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: middle; fill: var(--muted)" x="669.464569" y="230.330631" transform="rotate(-0 669.464569 230.330631)">x4</text>
</g>
</g>
<g id="text_22">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: middle; fill: var(--ink)" x="578.935261" y="243.040944" transform="rotate(-0 578.935261 243.040944)">layer 12's scalar, multiplied by</text>
</g>
</g>
<g id="matplotlib.axis_4">
<g id="ytick_7">
<g id="line2d_23">
<g>
<use xlink:href="#m31a93be521" x="479.353022" y="216.492038" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_23">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: end; fill: var(--muted)" x="472.353022" y="219.911334" transform="rotate(-0 472.353022 219.911334)">0.0</text>
</g>
</g>
<g id="ytick_8">
<g id="line2d_24">
<g>
<use xlink:href="#m31a93be521" x="479.353022" y="178.120743" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_24">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: end; fill: var(--muted)" x="472.353022" y="181.54004" transform="rotate(-0 472.353022 181.54004)">0.2</text>
</g>
</g>
<g id="ytick_9">
<g id="line2d_25">
<g>
<use xlink:href="#m31a93be521" x="479.353022" y="139.749449" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_25">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: end; fill: var(--muted)" x="472.353022" y="143.168746" transform="rotate(-0 472.353022 143.168746)">0.4</text>
</g>
</g>
<g id="ytick_10">
<g id="line2d_26">
<g>
<use xlink:href="#m31a93be521" x="479.353022" y="101.378155" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_26">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: end; fill: var(--muted)" x="472.353022" y="104.797452" transform="rotate(-0 472.353022 104.797452)">0.6</text>
</g>
</g>
<g id="ytick_11">
<g id="line2d_27">
<g>
<use xlink:href="#m31a93be521" x="479.353022" y="63.006861" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_27">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: end; fill: var(--muted)" x="472.353022" y="66.426158" transform="rotate(-0 472.353022 66.426158)">0.8</text>
</g>
</g>
<g id="ytick_12">
<g id="line2d_28">
<g>
<use xlink:href="#m31a93be521" x="479.353022" y="24.635567" style="fill: var(--muted); stroke: var(--muted); stroke-width: 0.8"/>
</g>
</g>
<g id="text_28">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: end; fill: var(--muted)" x="472.353022" y="28.054864" transform="rotate(-0 472.353022 28.054864)">1.0</text>
</g>
</g>
<g id="text_29">
<text style="font-size: 9px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: middle; fill: var(--ink)" x="452.168491" y="118.645238" transform="rotate(-90 452.168491 118.645238)">coordinate turn across layer 13</text>
</g>
</g>
<g id="line2d_29">
<path d="M 488.405953 34.052663
L 548.758825 40.149048
L 609.111697 52.836067
L 650.594328 68.259901
L 669.464569 77.691952
" clip-path="url(#p7eeb96b57c)" style="fill: none; stroke: var(--accent); stroke-width: 1.5; stroke-linecap: square"/>
<defs>
<path id="m76d05c59cb" d="M 0 2.5
C 0.663008 2.5 1.29895 2.236584 1.767767 1.767767
C 2.236584 1.29895 2.5 0.663008 2.5 0
C 2.5 -0.663008 2.236584 -1.29895 1.767767 -1.767767
C 1.29895 -2.236584 0.663008 -2.5 0 -2.5
C -0.663008 -2.5 -1.29895 -2.236584 -1.767767 -1.767767
C -2.236584 -1.29895 -2.5 -0.663008 -2.5 0
C -2.5 0.663008 -2.236584 1.29895 -1.767767 1.767767
C -1.29895 2.236584 -0.663008 2.5 0 2.5
z
" style="stroke: var(--accent)"/>
</defs>
<g clip-path="url(#p7eeb96b57c)">
<use xlink:href="#m76d05c59cb" x="488.405953" y="34.052663" style="fill: var(--accent); stroke: var(--accent)"/>
<use xlink:href="#m76d05c59cb" x="548.758825" y="40.149048" style="fill: var(--accent); stroke: var(--accent)"/>
<use xlink:href="#m76d05c59cb" x="609.111697" y="52.836067" style="fill: var(--accent); stroke: var(--accent)"/>
<use xlink:href="#m76d05c59cb" x="650.594328" y="68.259901" style="fill: var(--accent); stroke: var(--accent)"/>
<use xlink:href="#m76d05c59cb" x="669.464569" y="77.691952" style="fill: var(--accent); stroke: var(--accent)"/>
</g>
</g>
<g id="patch_28">
<path d="M 479.353022 216.492038
L 479.353022 20.798437
" style="fill: none; stroke: var(--muted); stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="patch_29">
<path d="M 479.353022 216.492038
L 678.5175 216.492038
" style="fill: none; stroke: var(--muted); stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="text_30">
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: start; fill: var(--muted)" x="496.704685" y="132.364126" transform="rotate(-0 496.704685 132.364126)">ordinary layers</text>
</g>
<g id="text_31">
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: start; fill: var(--ink)" x="560.928008" y="30.391261" transform="rotate(-0 560.928008 30.391261)">as trained</text>
</g>
<g id="text_32">
<text style="font-size: 8px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: start; fill: var(--ink)" x="621.28088" y="89.866767" transform="rotate(-0 621.28088 89.866767)">set to 1</text>
</g>
<g id="text_33">
<text style="font-weight: 600; font-size: 10px; font-family: 'IBM Plex Sans', 'DejaVu Sans', sans-serif; text-anchor: start" x="479.353022" y="14.798437" transform="rotate(-0 479.353022 14.798437)">The previous scalar sets part of it</text>
</g>
</g>
</g>
<defs>
<clipPath id="p38f4ca0da5">
<rect x="88.6375" y="20.798437" width="338.579613" height="195.6936"/>
</clipPath>
<clipPath id="p7eeb96b57c">
<rect x="479.353022" y="20.798437" width="199.164478" height="195.6936"/>
</clipPath>
</defs>
</svg></div>
<figcaption><b>Gemma 4 E2B, 2,000 tokens.</b> Left: each row changes the writes of layers 13 and 14 in one way.
Blue: coordinate turn across the two layers (1 − centered cosine, as in the track figure). Orange: company kept, the share of
each token's 10 nearest neighbours after layer 14 that match the unchanged model. Green: share of tokens whose
predicted next token is unchanged. Right: coordinate turn across layer 13 when layer 12's scalar is multiplied by
a factor; the band is the range of ordinary layers.</figcaption>
</figure>
<section class="prose">
<p>Read the left panel down the rows. The blue bars do not move: every change, even noise, leaves the turn as it
was. The orange and green bars fall as more of the content is replaced, down to almost no company kept when the
write is another text's or noise. So the size of the write sets the turn, and its content sets the company. Even
with every MLP unit held at its average, which adds the same vector to every token, the turn happens: whatever
else writes at those layers is large enough against the shrunken total. The same changes on Gemma 4 E4B also
leave its turn in place.</p>
<p>Part of the size comes from the layer before. A layer's own scalar cannot turn it, because multiplying everything
by one number leaves a cosine unchanged. But it shrinks the running total that the next layer writes onto. So
changing layer 12's scalar should change the turn at layer 13, and it does, smoothly (right panel). Even at four
times the trained value, layer 13 turns more than ordinary layers, so the size of its own write does the rest.</p>
</section>
<section class="prose" id="s-predict" aria-labelledby="h-predict">
<h3 class="tsub" id="h-predict">The same event in a larger model</h3>
<p>In Gemma 4 E2B, the smaller a layer's scalar, the sharper its turn. So before computing any geometry of the
larger Gemma 4 E4B, we read only its stored scalars and wrote down three predictions: its two sharpest turns lie in
layers 22–24; the scalar–turn rank correlation is at least 0.5; a block of shared coordinates starts at layer
23–25. All three held. <span class="chip pre">pre-registered · held-out model</span></p>
</section>
<figure>
<div class="scroll"><svg id="e4b" role="img" aria-label="Gemma 4 E4B: per-layer scalar and coordinate turn by layer"></svg></div>
<div class="legend"><span><i style="background:var(--c3)"></i>layer scalar (from the checkpoint)</span><span><i style="background:var(--warm)"></i>coordinate turn (1 − centered cosine across the layer)</span></div>
<figcaption><b>Gemma 4 E4B, one column per layer.</b> The green band is the predicted window, fixed before the turn
was measured.</figcaption>
</figure>
<section class="prose">
<p class="small">Caveats: the prediction window left out layer 1 (third-smallest scalar), a choice made after reading
the scalars. Both models are Gemma 4. A scalar meant to keep values in range is expected to be small where a layer
writes something huge; the test shows that this numerical device also marks the geometry.</p>
</section>
<section class="prose" id="s-behave" aria-labelledby="h-behave">
<h3 class="tsub" id="h-behave">Why the predictions tolerate it</h3>
<p>Does the turn matter for what the model predicts? A first test changed the turn layers' scalars, which sets how
sharp the turn is. We expected this to hurt the predictions more than the same change at ordinary layers. It hurt
less, even when the change was matched in absolute size <span class="chip fail">predicted · failed</span>. So
scalar changes do not show that the turn matters.</p>
<p>A cleaner test changes only the axes. At one layer's input we rotate every token's vector, keeping its length and
the layer's average, and measure how far the next-token predictions move. Just after the turn, rotating back toward
the pre-turn axes does as much damage as a random rotation. At ordinary layers, the axes of two layers back do far
less damage than random ones. So the layers after the turn get nothing from the old axes.</p>
<p>Across layers, a random rotation wrecks the predictions almost everywhere, but much less in a window that starts
inside the turn and fades over a few layers (figure). Gemma 4 E4B shows the same window, less deep. The few huge
coordinates do not cause it: holding them fixed while rotating the rest leaves the window unchanged.</p>
<p>The window comes from a feature of Gemma 4's design. Attention reads earlier tokens through <b>keys and
values</b>, summaries that each attention layer normally computes from the residual stream. In Gemma 4 the later
layers skip this and reuse the keys and values of earlier layers. Layers 13 and 14, the turn layers, are the last
two that compute their own, and every later layer reuses theirs. In E4B the turn layers are again the last two
(dotted lines in the figure). So a rotation after the turn cannot reach anything attention reads.</p>
<p>To test whether that is what protects the predictions, we rotated at layers before the turn while giving every
attention layer its keys and values from the unrotated model. We expected little effect, since a rotation there
also scrambles the input to the turn's large writes. Instead the damage almost vanished at every layer tried
(hollow points) <span class="chip pre">pre-registered</span>. The predictions can be rebuilt from keys and values
that the rotation cannot reach.</p>
</section>
<figure>
<div class="scroll"><svg xmlns:xlink="http://www.w3.org/1999/xlink" width="100%" style="max-width:900px;height:auto;display:block" role="img" aria-label="Damage to predictions from a random rotation of the residual stream at each layer input, Gemma 4 E2B and E4B" viewBox="0 0 656.676562 230.939063" xmlns="http://www.w3.org/2000/svg" version="1.1">
<metadata>
<rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<cc:Work>
<dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
<dc:date>2026-10-19T10:01:49.224941</dc:date>
<dc:format>image/svg+xml</dc:format>
<dc:creator>
<cc:Agent>
<dc:title>Matplotlib v3.10.3, https://matplotlib.org/</dc:title>
</cc:Agent>
</dc:creator>
</cc:Work>
</rdf:RDF>
</metadata>
<defs>
<style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style>
</defs>
<g id="figure_1">
<g id="patch_1">
<path d="M -0 230.939063
L 656.676562 230.939063
L 656.676562 0
L -0 0
L -0 230.939063
z
" style="fill: none"/>
</g>
<g id="axes_1">
<g id="patch_2">
<path d="M 32.636563 195.318438
L 331.589063 195.318438
L 331.589063 20.798437
L 32.636563 20.798437
L 32.636563 195.318438
z
" style="fill: none"/>
</g>
<g id="patch_3">
<path d="M 139.405312 195.318438
L 156.488312 195.318438
L 156.488312 20.798437
L 139.405312 20.798437
z
" clip-path="url(#pe412212815)" style="fill: var(--warm); opacity: 0.15"/>
</g>
<g id="matplotlib.axis_1">
<g id="xtick_1">
<g id="line2d_1">
<defs>
<path id="m95280dc717" d="M 0 0
L 0 3.5
" style="stroke: var(--muted); stroke-width: 0.8"/>
</defs>