% Based on the arxiv-paper-reader Beamer template.
% Build: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\R}{\mathbb R}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
\renewcommand{\arraystretch}{1.15}
\title[MesaNet]{MesaNet: Sequence Modeling by\\Locally Optimal Test-Time Training}
\subtitle{A reading of the ICLR 2026 paper}
\author{Paper: Johannes von Oswald, Nino Scherrer, et al.\\
\small Google, Google DeepMind, MIT CSAIL}
\date{\href{https://arxiv.org/abs/2506.05233v2}{arXiv:2506.05233v2, June 3, 2026}\\
\small Research discussion \textbullet\ October 20, 2026}
\begin{document}

\begin{frame}
\titlepage
\begin{center}
\color{AcademicTeal}Solve a small regression problem inside every sequence layer.
\end{center}
\end{frame}

\begin{frame}{Can a fixed memory learn the context more accurately?}
\begin{block}{The proposal}
Replace one online learning step with the solution of a cumulative,
regularized regression problem.
\end{block}
\begin{itemize}
\item Store two fixed-size sufficient statistics of past keys and values.
\item Solve only for the current query using conjugate gradient (CG).
\item Reuse gated linear attention structure for parallel training.
\end{itemize}
\medskip
\textbf{Three questions:} Does the memory improve? What does PPL conceal?
How much computation does the solver need?
\papersource{Paper, Secs. 1--2, PDF pp. 1--5.}
\end{frame}

\begin{frame}{Mesa solves regression over the retained history}
\textbf{DeltaNet:} one gradient step on the latest squared error
\[
\ell_t(\Phi)=\tfrac12\norm{v_t-\Phi k_t}^2,\qquad
\Phi_t=\Phi_{t-1}+\beta_t(v_t-\Phi_{t-1}k_t)k_t^\top.
\]
\textbf{Mesa:} solve a weighted regression on all retained associations
\[
\widehat\Phi_t=\arg\min_\Phi\left\{
\tfrac12\sum_{i\le t}\zeta_{ti}\beta_i\norm{v_i-\Phi k_i}^2
+\tfrac12\operatorname{Tr}(\Phi\Lambda\Phi^\top)\right\}.
\]
\begin{itemize}
\item ``Local'' refers to a layer/head objective; it has a unique optimum.
\item Outer next-token training learns which keys, values and gates to produce.
\end{itemize}
\papersource{Paper, Eqs. (3)--(8), PDF pp. 3--4; Appendix B, pp. 21--23.
Input gates are written explicitly here.}
\end{frame}

\begin{frame}{The sequence mixer changes within a common backbone}
\centering
\includegraphics[width=.96\linewidth,height=.48\textheight,keepaspectratio]{figures/architecture.pdf}
\medskip
\begin{itemize}
\item Pre-norm residual blocks alternate a sequence mixer and a SwiGLU MLP.
\item Mesa produces $k,q,v$, input/forget gates, and normalized head outputs.
\item Hawk-Mesa alternates Hawk and Mesa blocks.
\end{itemize}
\papersource{Original Fig. 1, PDF p. 2; Appendix E, pp. 28--30.}
\end{frame}

\begin{frame}{A positive regularizer makes the regression well defined}
\[
\zeta_{ti}=\prod_{s=i+1}^{t}\gamma_s,\quad i\le t,
\qquad 0\le\gamma_s,\beta_s\le1,\quad \Lambda\succ0.
\]
\[
0=\nabla_\Phi\mathcal L_t
=\Phi\underbrace{\left(\sum_{i\le t}\zeta_{ti}\beta_i k_i k_i^\top+
\Lambda\right)}_{H_t+\Lambda}
-\underbrace{\sum_{i\le t}\zeta_{ti}\beta_i v_i k_i^\top}_{G_t}.
\]
\begin{block}{The solution}
\[
\widehat\Phi_t=G_t(H_t+\Lambda)^{-1},
\qquad o_t=\widehat\Phi_tq_t.
\]
\end{block}
$k,q\in\R^{n_a}$, $v,o\in\R^{n_v}$; $G,\Phi\in\R^{n_v\times n_a}$,
$H,\Lambda\in\R^{n_a\times n_a}$.
\papersource{Paper, Eqs. (4)--(8), PDF pp. 3--4; derivation in Appendix D, pp. 25--26.}
\end{frame}

\begin{frame}{Two statistics replace the full history}
\[
\boxed{\begin{aligned}
G_t&=\gamma_tG_{t-1}+\beta_tv_tk_t^\top,\\
H_t&=\gamma_tH_{t-1}+\beta_tk_tk_t^\top,\\
(H_t+\Lambda)q_t^*&=q_t,\qquad o_t=G_tq_t^*.
\end{aligned}}
\]
\begin{itemize}
\item $G_t$: associations between keys and values.
\item $H_t$: repeated and correlated directions in key space.
\item $\Lambda$: fixed over time, so forgetting does not erase regularization.
\end{itemize}
\medskip
For $n_a=n_v=d$, persistent state is $2d^2$ numbers per head and sequence.
\par Core cost per token: $O((K+1)d^2)$ for $K$ CG iterations.
\papersource{Paper, Eqs. (7)--(8), PDF p. 4; Table 5, p. 29.}
\end{frame}

\begin{frame}{CG solves a query direction without forming an inverse}
\begin{columns}[T]
\begin{column}{.53\linewidth}
\[
A_tq_t^*=q_t,\qquad A_t=H_t+\Lambda.
\]
\begin{itemize}
\item Main cost per iteration: $A_tp$.
\item Other operations: inner products and vector updates.
\item Main experiments: 30 iterations.
\end{itemize}
\end{column}
\begin{column}{.43\linewidth}
\begin{block}{Adaptive stopping}
\[
r_j=q_t-A_tx_j
\]
\[
\norm{r_j}\le\epsilon\norm{r_0}.
\]
Relative to the initial residual, with an iteration cap.
\end{block}
\end{column}
\end{columns}
\medskip
\textbf{Interpretation:} computation depends on the head and context through
the linear system. Finite CG approximates the exact optimum.
\papersource{Paper, Appendix C, Algorithm 1, PDF pp. 24--25; Sec. 5, pp. 9--10.}
\end{frame}

\begin{frame}{Each CG iteration can reuse chunked linear attention}
For a chunk starting after token $c$, absorb $\sqrt{\beta_i}$ into $k_i,v_i$:
\[
o_t=\zeta_{tc}G_cq_t^*
+\sum_{i=c+1}^t\zeta_{ti}v_i(k_i^\top q_t^*).
\]
The expensive solver operation has the same structure:
\[
(H_t+\Lambda)p_t
=\zeta_{tc}H_cp_t+
\sum_{i=c+1}^t\zeta_{ti}k_i(k_i^\top p_t)+\Lambda p_t.
\]
\begin{itemize}
\item Store full $G,H$ only at chunk boundaries.
\item Process token positions within a chunk using matrix multiplications.
\item CG rounds remain sequential; positions within a round are parallel.
\end{itemize}
\papersource{Expanded from Eq. (8), PDF p. 4; Appendix D, pp. 25--28.
Cross-chunk forgetting factors are retained explicitly.}
\end{frame}

\begin{frame}{Training also needs a linear solve in the backward pass}
Let $g_t=\partial\mathcal J/\partial o_t$ and solve
\[
(H_t+\Lambda)u_t=G_t^\top g_t.
\]
Then
\[
\frac{\partial\mathcal J}{\partial q_t}=u_t,\quad
\frac{\partial\mathcal J}{\partial G_t}=g_tq_t^{*\top},\quad
\frac{\partial\mathcal J}{\partial\Lambda_{ii}}=-u_{t,i}q^*_{t,i}.
\]
\begin{itemize}
\item The paper differentiates the solution map, assuming an accurate solve.
\item Key/query normalization and $\Lambda_{ii}\ge0.25$ aid stability.
\item Pallas solver matmuls use FP32; surrounding activations use BF16.
\end{itemize}
\papersource{Paper, Appendix D, PDF pp. 26--28; G.5, pp. 33--34; J, p. 37.
Gradient symbols renamed for clarity.}
\end{frame}

\begin{frame}{The study controls architecture, data and tokenizer}
\begin{center}
\begin{tabular}{lccc}
\toprule
Approximate size & Blocks & Heads & Embedding size\\
\midrule
140M & 14 & 6 & 768\\
440M & 28 & 8 & 1024\\
940M (``1B'') & 28 & 12 & 1536\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item SlimPajama: 15B / 50B tokens; GPT-2 tokenizer; train length 2048.
\item Same data order and backbone; head key dimension 128.
\item Learning rates tuned by model; SWA references use a fixed rate.
\end{itemize}
\textbf{Comparison scope:} controlled sequence mixers, with matched parameters
and data. FLOPs are not matched; some baseline parameterizations are adapted.
\papersource{Paper, Sec. 4, PDF pp. 6--7; Appendices E and G, pp. 28--34.
Result tables use approximate 145M / 400M / 1B labels.}
\end{frame}

\begin{frame}{Average PPL reaches the Transformer baseline}
\centering
\textbf{1B models, 50B training tokens, evaluation length 2048}
\medskip
\begin{tabular}{lrrr}
\toprule
Model & SlimPajama $\downarrow$ & Qasper $\downarrow$ & Six-dataset avg. $\downarrow$\\
\midrule
Mamba2 & 11.39 & 14.02 & 14.58\\
DeltaNet & 11.01 & 13.44 & 14.05\\
Gated DeltaNet & 10.89 & 13.28 & 13.87\\
\textbf{MesaNet} & \textbf{10.83} & \textbf{13.13} & \textbf{13.79}\\
Hawk-Mesa & 10.78 & 13.20 & 13.75\\
SWA-1024 & 11.00 & 13.11 & 13.97\\
Transformer & 10.86 & 12.86 & 13.79\\
\bottomrule
\end{tabular}
\medskip
\par Mesa ties the average while individual datasets differ.
\papersource{Paper, Table 2, PDF p. 7. Average is the arithmetic mean of six dataset PPLs,
not pooled-token PPL.}
\end{frame}

\begin{frame}{Position-wise loss reveals where the advantage comes from}
\centering
\includegraphics[width=.82\linewidth,height=.66\textheight,keepaspectratio]{figures/position-nll.pdf}
\par $\Delta\mathrm{NLL}_k=\mathrm{NLL}^{\rm model}_k-\mathrm{NLL}^{\rm MHA}_k$.
Below zero is better.
\par Mesa's early-token advantage extends beyond 512; later tokens favor MHA.
\papersource{Original Fig. 3, PDF p. 7. 1B / 50B; right panel uses a log token-position axis.}
\end{frame}

\begin{frame}{Window controls separate different benchmark sensitivities}
\begin{columns}[T]
\begin{column}{.48\linewidth}
\begin{block}{Global subset}
Lambada, HellaSwag, RACE-M/H.
\end{block}
Scores improve as short windows expand; short-window models do not beat MHA.
\end{column}
\begin{column}{.48\linewidth}
\begin{block}{Local subset}
PIQA, WinoGrande, ARC-E/C, SIQA, BoolQ, OBQA, StoryCloze.
\end{block}
At least one of these window-sensitivity conditions fails.
\end{column}
\end{columns}
\medskip
\begin{center}
\begin{tabular}{lrr}
\toprule
1B / 50B & Global accuracy & Local accuracy\\
\midrule
SWA-4 & 31.20 & 49.62\\
MesaNet & 45.03 & 50.49\\
Transformer & 45.54 & 51.62\\
\bottomrule
\end{tabular}
\end{center}
\papersource{Paper, Table 3(b), p. 8; Appendix L.2.1, pp. 44--46.
Empirical grouping for these scales; the SWA window is per layer.}
\end{frame}

\begin{frame}{Recall improves, but the gap to full attention remains}
\begin{columns}[T]
\begin{column}{.51\linewidth}
\begin{tabular}{lr}
\toprule
1B / 50B model & Recall avg. (\%)\\
\midrule
DeltaNet & 40.46\\
Gated DeltaNet & 39.54\\
\textbf{MesaNet} & \textbf{41.79}\\
SWA-1024 & 40.84\\
Transformer & 52.27\\
\bottomrule
\end{tabular}
\end{column}
\begin{column}{.45\linewidth}
\begin{block}{Mesa relative to baselines}
$+2.25$ points vs.\ Gated DeltaNet.\\
$-10.48$ points vs.\ Transformer.
\end{block}
FDA: 32.67 vs.\ 70.96.\\
SQuAD: 46.25 vs.\ 46.92.
\end{column}
\end{columns}
\medskip
\begin{itemize}
\item Six recall tasks; inputs truncated to 2000 tokens.
\item Greedy output, at most 48 tokens; case-insensitive target containment.
\item Target position helps explain why tasks show different gaps.
\end{itemize}
\papersource{Paper, Table 3(b), p. 8; Appendix L.2.2, Tables 15--16, pp. 47--48.}
\end{frame}

\begin{frame}{Few-shot learning depends strongly on the task}
\centering
\textbf{1B models trained on 50B tokens}
\medskip
\begin{tabular}{lrr}
\toprule
Model & Word scrambling & Translation\\
 & 100-shot accuracy (\%) & 50-shot BLEU-sb\\
\midrule
Gated DeltaNet & 8.90 & 8.53\\
\textbf{MesaNet} & \textbf{10.10} & 8.17\\
Hawk-Mesa & 8.61 & 8.81\\
SWA-1024 & 6.66 & 14.17\\
Transformer & 6.98 & 13.61\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Mesa leads on the two reported word-transformation tasks.
\item Translation into English favors attention; Mesa also trails Gated DeltaNet.
\end{itemize}
\papersource{Paper, Table 3(b), p. 8; Tables 17--18, pp. 49--50.
Two-task accuracy mean; three-language BLEU-sb mean.}
\end{frame}

\begin{frame}{Length extrapolation and long-range recall are distinct}
\begin{columns}[T]
\begin{column}{.51\linewidth}
\includegraphics[width=\linewidth,height=.60\textheight,keepaspectratio]{figures/length-ppl.pdf}
\end{column}
\begin{column}{.45\linewidth}
\textbf{32K average PPL, 1B / 50B}
\medskip
\begin{tabular}{lr}
\toprule
Gated DeltaNet & 12.44\\
MesaNet & 12.63\\
SWA-1024 & 12.75\\
Hawk-Mesa & 15.09\\
Transformer & 588.41\\
\bottomrule
\end{tabular}
\medskip
\par Mesa stays stable far beyond its 2K training length.
\end{column}
\end{columns}
\medskip
SWA remains competitive. The MHA failure concerns this 2K-trained baseline.
\papersource{Original Fig. 4, p. 8: mean-so-far PPL on three datasets.
Table 12, p. 43: four-dataset mean including PG19; these averages differ.}
\end{frame}

\begin{frame}{A strong noise-needle score can hide an essay weakness}
\centering
\textbf{NIAH accuracy (\%), 1B / 50B}
\medskip
\begin{tabular}{lrrrr}
\toprule
Model & Noise 2K & Noise 4K & Essay 2K & Essay 4K\\
\midrule
Gated DeltaNet & 98.3 & 96.3 & 52.1 & 11.0\\
MesaNet & \textbf{99.5} & 95.1 & 66.8 & 17.9\\
Hawk-Mesa & 97.6 & 65.3 & \textbf{90.9} & \textbf{55.5}\\
Transformer & 99.7 & 0.0 & 98.2 & 0.0\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Means across 9 key/value types; 1000 samples per setting.
\item Mesa Essay 2K: $66.8\pm28.9$ across types, versus Noise $99.5\pm0.5$.
\item 4K is outside the training length; the haystack changes model rankings.
\end{itemize}
\papersource{Paper, Appendix L.3, pp. 50--51; Table 19, p. 51.
Standard deviations are across key/value types, not training runs.}
\end{frame}

\begin{frame}{Dynamic stopping preserves the later-token predictions}
\centering
\includegraphics[width=.98\linewidth,height=.61\textheight,keepaspectratio]{figures/cg-loss.pdf}
\begin{itemize}
\item Uniformly reducing iterations hurts later positions more.
\item A residual criterion allocates more work where the solve needs it.
\end{itemize}
\papersource{Original Fig. 5, right side, PDF p. 10. 1B / 50B, trained with 30 CG steps.
Left/middle: NLL difference from 30 steps; right: average iterations.}
\end{frame}

\begin{frame}{About 9 average iterations can retain 30-step performance}
\begin{columns}[T]
\begin{column}{.65\linewidth}
\includegraphics[width=\linewidth,height=.61\textheight,keepaspectratio]{figures/cg-benchmarks.pdf}
\end{column}
\begin{column}{.31\linewidth}
\begin{block}{At $\epsilon=10^{-4}$}
Average CG steps:\\
\[
30\longrightarrow \approx9
\]
Global accuracy:\\
45.03 $\rightarrow$ 45.03
\end{block}
About 70\% fewer iterations; end-to-end speedup is a separate measurement.
\end{column}
\end{columns}
\papersource{Original Fig. 5, left side, PDF p. 10, and Sec. 5.
Some heads need many more iterations than others.}
\end{frame}

\begin{frame}{Parallel kernels make the added FLOPs more practical}
\centering
\includegraphics[width=.90\linewidth,height=.60\textheight,keepaspectratio]{figures/train-throughput.pdf}
\begin{itemize}
\item 1B models on H100; sequence length $\times$ batch size is about 32K tokens.
\item Mesa is competitive, while GLA remains faster.
\end{itemize}
\papersource{Original Fig. 2, right panel, PDF p. 5.
FlashAttention-2 MHA and Triton recurrent implementations; fixed 15/30 CG steps.}
\end{frame}

\begin{frame}{Decode throughput depends on memory traffic and hardware}
\begin{columns}[T]
\begin{column}{.57\linewidth}
\includegraphics[width=\linewidth,height=.65\textheight,keepaspectratio]{figures/decode-length.png}
\end{column}
\begin{column}{.39\linewidth}
\begin{itemize}
\item H100, 1B models, batch size 8.
\item Recurrent throughput is nearly flat as generated length increases.
\item Mesa can approach Gated DeltaNet despite more arithmetic.
\item GLA retains the highest throughput here.
\end{itemize}
\end{column}
\end{columns}
\papersource{Original Fig. 6, upper-right panel, PDF p. 35; Appendix G.7.
Five timing repetitions; fixed CG budgets. TPU results have different relative timings.}
\end{frame}

\begin{frame}{The gain is better use of finite memory, at a compute cost}
\begin{enumerate}
\item \textbf{Mechanism:} cumulative ridge regression, two sufficient statistics,
and a chunk-parallel solver scale the Mesa layer.
\medskip
\item \textbf{Evidence:} strong average PPL and useful recall gains coexist with
a substantial gap to full attention on recall.
\medskip
\item \textbf{Tradeoff:} more solver work and state; accuracy, latency and
context use need separate evaluation.
\end{enumerate}
\begin{block}{A follow-up question}
At equal latency or FLOPs, should extra budget buy a more accurate solve,
a larger memory, or an attention component?
\end{block}
\papersource{Paper, Secs. 4--6 and Appendices I--J. The follow-up question is reader analysis.}
\end{frame}

\begin{frame}{Source and discussion guide}
\textbf{Johannes von Oswald, Nino Scherrer, et al.}\\
\emph{MesaNet: Sequence Modeling by Locally Optimal Test-Time Training.}\\
ICLR 2026; arXiv:2506.05233v2 (June 3, 2026).
\medskip
\par\href{https://arxiv.org/abs/2506.05233v2}{https://arxiv.org/abs/2506.05233v2}
\medskip
\begin{itemize}
\item Mechanism: Sec. 2 and Appendices C--E.
\item Main evidence: Tables 2--3, Figs. 3--5.
\item Reproduction details: Appendices G, L, M and N.
\item Open questions: Sec. 6 and Appendices I--J.
\end{itemize}
\medskip
Figures are original author assets. Selected tables were re-typeset from the paper.
The following slides provide optional technical and experimental detail.
\end{frame}

\appendix

\begin{frame}{Backup: correlated keys illustrate the benefit of regression}
\[
k_1=(1,0)^\top,\quad k_2=(1,1)^\top/\sqrt2,\quad v_1=1,\ v_2=0.
\]
No forgetting, input weights 1, $\Lambda=\lambda I$. Read both keys after two writes:
\begin{center}
\begin{tabular}{l@{\hspace{8mm}}c@{\hspace{8mm}}c@{\hspace{8mm}}c}
\toprule
Rule & Map $\Phi$ & Read $k_1$ & Read $k_2$\\
\midrule
Additive GLA & $(1,0)$ & 1 & $1/\sqrt2$\\
Delta, step size 1 & $(1/2,-1/2)$ & $1/2$ & 0\\
Mesa, $\lambda\to0^+$ & $(1,-1)$ & 1 & 0\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Correcting the latest error can change an earlier association.
\item The cumulative solution accounts for key correlations.
\item Positive regularization introduces shrinkage; capacity still limits recall.
\end{itemize}
\papersource{Teaching example derived from the paper's updates, not a reported experiment.}
\end{frame}

\begin{frame}{Backup: why the earlier RLS implementation was restrictive}
Without forgetting, set $R_0=\Lambda$ and $R_t=R_{t-1}+k_tk_t^\top$:
\[
R_t^{-1}=R_{t-1}^{-1}-
\frac{R_{t-1}^{-1}k_tk_t^\top R_{t-1}^{-1}}
{1+k_t^\top R_{t-1}^{-1}k_t}.
\]
\begin{itemize}
\item Efficient sequential inverse update, but limited training parallelism.
\item Forgetting introduces inverse scaling by $1/\gamma_t$.
\item A regularizer stored only in the initial state decays with the history.
\end{itemize}
\begin{block}{The new decomposition}
Accumulate $H_t$ directly; add a fixed $\Lambda$ at solve time.
\end{block}
\papersource{Paper, Appendix H, PDF pp. 34--36. Warm starts and hybrid solvers remain future work.}
\end{frame}

\begin{frame}{Backup: synthetic tasks favor Mesa's context learning}
\begin{columns}[T]
\begin{column}{.46\linewidth}
\textbf{MAD mean accuracy (\%)}
\medskip
\begin{tabular}{l@{\hspace{7mm}}r}
\toprule
Gated DeltaNet & 71.0\\
xLSTM & 73.2\\
Hawk-Mesa & 73.5\\
Transformer & 75.8\\
MesaNet & 76.1\\
\bottomrule
\end{tabular}
\medskip
\par Tasks cover recall, memorization, copying and compression.
\end{column}
\begin{column}{.50\linewidth}
\includegraphics[width=\linewidth,height=.57\textheight,keepaspectratio]{figures/regbench.png}
\par RegBench tests new automata: infer a grammar from context.
\end{column}
\end{columns}
\papersource{Paper, Table 9 and original Fig. 9, PDF p. 39; settings in Appendix F, pp. 30--31.
These models are trained on the synthetic tasks.}
\end{frame}

\begin{frame}{Backup: parity exposes a tension with the regression view}
\begin{columns}[T]
\begin{column}{.55\linewidth}
\includegraphics[width=\linewidth,height=.58\textheight,keepaspectratio]{figures/parity-accuracy.pdf}
\end{column}
\begin{column}{.41\linewidth}
\begin{itemize}
\item Train on bitstrings of length 40; evaluate up to 256.
\item Negative forget gates enable parity in this experiment.
\item $\Lambda$ starts at $50I$, with a lower bound of 49.
\end{itemize}
\end{column}
\end{columns}
\medskip
Large regularization makes the readout close to GLA. Negative gates also
remove the usual nonnegative weighted-regression interpretation.
\papersource{Original Fig. 8, right panel, PDF p. 36; Appendix I, pp. 36--37.
An exploratory variant, not the main language model.}
\end{frame}

\begin{frame}{Backup: too few training iterations change the learned model}
\begin{columns}[T]
\begin{column}{.58\linewidth}
\includegraphics[width=\linewidth,height=.65\textheight,keepaspectratio]{figures/cg-training.pdf}
\end{column}
\begin{column}{.38\linewidth}
\begin{itemize}
\item About 400M parameters, 15B training tokens.
\item 10--25 training steps cause small changes relative to 30.
\item Five steps incur a clear loss penalty.
\item Extra inference steps only partly recover the 5-step model.
\end{itemize}
\end{column}
\end{columns}
\papersource{Original Fig. 13, PDF p. 51; Appendix M.
Training-budget ablation differs from reducing steps on a 30-step-trained model.}
\end{frame}

\begin{frame}{Backup: heads develop different solver requirements}
\centering
\includegraphics[width=.99\linewidth,height=.40\textheight,keepaspectratio]{figures/heads-layer16.png}
\begin{itemize}
\item Example: layer 16 of a medium MesaNet trained on 50B tokens.
\item Weak forgetting can accompany growing condition numbers and more CG steps.
\item Other heads maintain lower, nearly constant solve costs.
\end{itemize}
\textbf{Implementation detail:} identity query transformation gives GLA;
zero CG steps depend on the chosen initialization.
\papersource{Original Fig. 14, layer-16 row, PDF p. 52; averaged over 500 sequences.
Internal diagnostic uses tolerance $10^{-3}$. Initialization caveat: Appendix C, p. 24.}
\end{frame}

\end{document}