% Based on the arxiv-paper-reader Beamer template.
% Compile with the accompanying build_slides.py (pdfLaTeX, no shell escape).
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\DP}{\mathrm{DP}}
\newcommand{\DN}{\mathrm{DN}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\norm}[1]{\left\|#1\right\|_2}
\newcommand{\takeaway}[1]{\begin{block}{Key point}#1\end{block}}

\title[DeltaProduct]{DeltaProduct}
\subtitle{Improving State-Tracking in Linear RNNs\\via Householder Products}
\author{Paper by Julien Siems, Timur Carstensen, Arber Zela,\\
Frank Hutter, Massimiliano Pontil, and Riccardo Grazzi}
\date{NeurIPS 2025 \quad arXiv:2502.10297v7\\
Research discussion prepared on 20 October 2026}

\begin{document}

\begin{frame}
\titlepage
\vspace{-5mm}
\begin{center}
\color{AcademicTeal}More memory updates per token enable richer state transitions.
\end{center}
\end{frame}

\begin{frame}{The transition matrix limits what memory can do}
\[
H_i=A(x_i)H_{i-1}+B(x_i),\qquad H_i\in\R^{n\times d}
\]
\begin{center}
\begin{tabular}{lll}
\toprule
Structure of $A$ & Main capability & Main trade-off\\
\midrule
Diagonal & Coordinate-wise retention & Limited state mixing\\
$I-\beta kk^\top$ & One directional correction & One factor per token\\
Householder product & Composed transformations & More internal updates\\
Dense & General state transformations & Expensive recurrence\\
\bottomrule
\end{tabular}
\end{center}
\medskip
State tracking requires composing an ordered sequence of operations:
\[
y_i=x_i\cdot x_{i-1}\cdots x_1.
\]
\papersource{Paper, Secs. 1--2, PDF pp. 1--3. The group experiments use permutations.}
\end{frame}

\begin{frame}{DeltaNet corrects one key--value association}
The matrix state is a linear predictor from keys to values:
\[
\mathcal L_i(H)=\frac12\norm{H^\top k_i-v_i}^2.
\]
One gradient step gives
\[
H_i=H_{i-1}+
\beta_i k_i\underbrace{(v_i-H_{i-1}^\top k_i)^\top}_{\text{prediction error}}.
\]
\begin{itemize}
\item $k_i\in\R^n$, $\norm{k_i}=1$; $v_i\in\R^d$; $H_i\in\R^{n\times d}$.
\item With $\beta_i=1$, the updated memory satisfies $H_i^\top k_i=v_i$.
\item Directions orthogonal to $k_i$ retain their previous contents.
\end{itemize}
\papersource{Paper, Sec. 2, Eq. (2) and the online-learning derivation, PDF p. 3.}
\end{frame}

\begin{frame}{DeltaProduct generates several associations per token}
For each input $x_i$, generate $n_h$ distinct triples:
\[
k_{i,j}=\frac{\psi(W_jx_i)}{\norm{\psi(W_jx_i)}},\quad
v_{i,j}=V_jx_i,\quad \beta_{i,j}=\phi(U_jx_i).
\]
Then update the same memory sequentially:
\[
H_{i,j}=(I-\beta_{i,j}k_{i,j}k_{i,j}^\top)H_{i,j-1}
+\beta_{i,j}k_{i,j}v_{i,j}^\top.
\]
\[
H_{i,0}=H_{i-1}\quad\longrightarrow\quad H_{i,n_h}=H_i.
\]
\begin{itemize}
\item $\psi=\mathrm{SiLU}$; $\phi=\mathrm{sigmoid}$ or $2\,\mathrm{sigmoid}$.
\item $n_h$ counts updates per token, not attention heads.
\item Read the final state using $H_i^\top q_i$, then decode.
\end{itemize}
\papersource{Paper, Sec. 4, PDF p. 4.}
\end{frame}

\begin{frame}{Update order changes both transition and writing}
Write $F_j=I-\beta_jk_jk_j^\top$ and $C_j=\beta_jk_jv_j^\top$
for a single token. With two updates,
\[
H_i=\underbrace{F_2F_1}_{A_i}H_{i-1}
+\underbrace{F_2C_1+C_2}_{B_i}.
\]
\[
F_2F_1=I-\beta_1k_1k_1^\top-\beta_2k_2k_2^\top
+\beta_1\beta_2(k_2^\top k_1)k_2k_1^\top.
\]
\begin{itemize}
\item The cross term can make the transition non-symmetric.
\item Later updates transform earlier writes.
\item The structural bound is $\operatorname{rank}(A_i-I)\le n_h$.
\end{itemize}
\takeaway{The product expands the changed subspace; $A_i$ itself can already be full rank.}
\papersource{Paper, Eq. (3), PDF pp. 4--5. Product order is written explicitly here.}
\end{frame}

\begin{frame}{Two reflections can implement a rotation}
\begin{columns}[T]
\begin{column}{0.47\textwidth}
For $F=I-\beta kk^\top$:
\[
Fk=(1-\beta)k.
\]
\begin{itemize}
\item $\beta=0$: identity.
\item $\beta=1$: projection.
\item $\beta=2$: reflection.
\end{itemize}
\medskip
Directions orthogonal to $k$ keep eigenvalue 1.
\end{column}
\begin{column}{0.50\textwidth}
\textbf{Teaching example: count modulo 3}
\[
k_1=\binom10,\quad
k_2=\binom{1/2}{\sqrt3/2},\quad \beta_1=\beta_2=2.
\]
\[
F_2F_1=
\begin{pmatrix}
-1/2&-\sqrt3/2\\
\sqrt3/2&-1/2
\end{pmatrix}.
\]
A $120^\circ$ rotation cycles through three distinct states.
\end{column}
\end{columns}
\medskip
\alert{A product may have complex eigenvalues, even though each factor is symmetric.}
\papersource{Paper, Fig. 3, PDF p. 5; Appendix A, pp. 17--18. Numerical example derived for this discussion.}
\end{frame}

\begin{frame}{Products preserve a useful norm guarantee}
With unit keys and $\beta_{i,j}\in[0,2]$,
\[
\norm{F_{i,j}}\le1
\quad\Rightarrow\quad
\norm{A_i}\le1
\quad\Rightarrow\quad
\norm{A_t\cdots A_1}\le1.
\]
\begin{itemize}
\item The homogeneous recurrence cannot amplify old-state perturbations.
\item With a gate, $A_i=g_iF_{i,n_h}\cdots F_{i,1}$, $g_i\in[0,1]$.
\item Additive inputs can still accumulate; this is a transition-product guarantee.
\end{itemize}
\medskip
\textbf{Why not add the corrections?} For identical keys and $\beta_1=\beta_2=2$:
\[
(I-2kk^\top)^2=I,\qquad I-4kk^\top\text{ has eigenvalue }-3.
\]
\papersource{Paper, Sec. 4, PDF p. 5; Appendix B.6, pp. 27--28. Last example is explanatory algebra.}
\end{frame}

\begin{frame}{More updates reduce the depth needed in constructions}
\textbf{Theorem 1:} sufficient configurations for the word problem of $S_n$.
\begin{center}
\begin{tabular}{lll}
\toprule
Layers & Updates per token & Main construction cost\\
\midrule
1 & $n_h=n-1$ & Direct permutation factorization\\
3 & Any $n_h>1$ & Large lookup MLP\\
4 & $n_h=1$ & Larger lookup MLP\\
\bottomrule
\end{tabular}
\end{center}
\medskip
A coordinate swap is a reflection:
\[
k=(e_a-e_b)/\sqrt2,\qquad \beta=2.
\]
Any permutation of $n$ objects needs at most $n-1$ swaps.
\papersource{Paper, Theorem 1, PDF p. 5; proof in Appendix B.2, pp. 21--22.}
\end{frame}

\begin{frame}{A small layer count can hide an enormous lookup}
Let $m=\lceil(n-1)/n_h\rceil$. The low-depth construction:
\begin{enumerate}
\item Computes position modulo $2m$.
\item Stores the last $2m$ tokens and looks up a factorization.
\item Executes delayed transformations; the decoder completes the prefix.
\end{enumerate}
\[
\text{Lookup size}=2m(n!)^{2m}.
\]
For $S_5$ and $n_h=2$: $4\cdot120^4=829{,}440{,}000$ combinations.
\takeaway{Existence uses task-dependent state capacity and sufficiently expressive parameter maps and decoders.}
\papersource{Paper, Theorem 1 and Appendix B.1--B.2, PDF pp. 19--22. Lookup count calculated from the stated formula.}
\end{frame}

\begin{frame}{Gates add reset operations for regular languages}
\[
H_{i,0}=g_iH_{i-1},\qquad
H_{i,j}=F_{i,j}H_{i,j-1}+C_{i,j}.
\]
\begin{itemize}
\item Householder products implement permutations.
\item $g_i=0$ removes the old state; writing sets a chosen reset state.
\item Krohn--Rhodes decomposition connects permutation-reset automata.
\end{itemize}
\begin{block}{Theorem 2}
For any $n_h\ge1$, any regular language can be recognized with a finite,
language-dependent number of Gated DeltaProduct layers.
\end{block}
Products of RWKV-7 matrices can also represent copy transitions, at the cost
of losing the same universal norm guarantee.
\papersource{Paper, Theorem 2 and Remark 1, PDF pp. 5--6; Appendix B.3--B.4, pp. 23--24.}
\end{frame}

\begin{frame}{Throughput depends on how parameters are matched}
\begin{columns}[c]
\begin{column}{0.55\textwidth}
\begin{itemize}
\item Expand each token into $n_h$ internal key/value updates.
\item Use gates $[g_i,1,\ldots,1]$ and keep every $n_h$-th output.
\item Reuse a chunkwise DeltaNet-style kernel.
\item At fixed head shape, recurrent work scales with $n_h$; state size stays fixed.
\end{itemize}
\medskip
Non-power-of-two head dimensions may be padded by the kernel.
\end{column}
\begin{column}{0.41\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.66\textheight,keepaspectratio]{figures/fig-04-throughput.pdf}
\end{column}
\end{columns}
\papersource{Paper, Sec. 5.1 and Fig. 4, PDF pp. 6--7. 1.3B parameters, H100. Top: head count; bottom: head dimension.}
\end{frame}

\begin{frame}{The experiments test four distinct questions}
\begin{center}
\begin{tabular}{>{\raggedright\arraybackslash}p{0.22\textwidth}
>{\raggedright\arraybackslash}p{0.32\textwidth}
>{\raggedright\arraybackslash}p{0.34\textwidth}}
\toprule
Experiment & Comparison & Outcome measured\\
\midrule
Group tracking & More updates vs.\ more layers & Accuracy by position;\newline train 128, test 512\\[2mm]
Formal languages & $n_h$ and coefficient range & Scaled accuracy, length 40--256\\[2mm]
Language models & Parameter-matched models & Perplexity and six-task accuracy\\[2mm]
Long context & Beyond 4096 training tokens & Token loss; state behavior\\
\bottomrule
\end{tabular}
\medskip
\end{center}
Matched parameters do not imply matched FLOPs, state capacity, or wall time.
\papersource{Paper, Sec. 5 and Appendix C, PDF pp. 7--10 and 28--39.}
\end{frame}

\begin{frame}{More updates delay failure on group tracking}
\centering
\includegraphics[width=0.94\linewidth,height=0.62\textheight,keepaspectratio]{figures/fig-05-state-tracking.pdf}
\par\smallskip
\begin{minipage}{0.96\linewidth}
Top: one layer, varying $n_h$. Bottom: $n_h=1$, varying depth.
\alert{Even $S_5$ with $n_h=4$ degrades strongly near length 512.}
\end{minipage}
\papersource{Paper, Fig. 5, PDF p. 7. Dashed vertical line: training length 128.}
\end{frame}

\begin{frame}{A learned head resembles a 3D rotation mechanism}
\begin{columns}[c]
\begin{column}{0.38\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.48\textheight,keepaspectratio]{figures/fig-07-beta.pdf}
\end{column}
\begin{column}{0.38\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.48\textheight,keepaspectratio]{figures/fig-07-pca.pdf}
\end{column}
\end{columns}
\begin{itemize}
\item $S_4$ is the rotation group of the cube.
\item One head learns both $\beta$ values close to 2.
\item Three principal components explain over 95\% of key variance.
\end{itemize}
This supports the two-reflection explanation for the selected head.
\papersource{Paper, Fig. 7, PDF p. 8; Fig. 13, p. 30, identifies head 6 among 12 heads.}
\end{frame}

\begin{frame}{Language-model comparisons use three budget levels}
\begin{center}
\begin{tabular}{ccc}
\toprule
Parameters & FineWeb training tokens & Context length\\
\midrule
213M & 19B & 4096\\
392M & 35B & 4096\\
805M & 55B & 4096\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Table 3 matches parameters through head dimension.
\item Table 4 uses a separate set matched through head count.
\item Average accuracy: LAMBADA, PIQA, HellaSwag, Winogrande, ARC-e, ARC-c.
\item HellaSwag and ARC-c use normalized accuracy; PPL is excluded from the average.
\end{itemize}
\papersource{Paper, Tables 3--4, PDF p. 32; training setup in Appendix C.3.1, p. 31.}
\end{frame}

\begin{frame}{At 805M, three updates improve language modeling}
\textbf{55B FineWeb tokens, 4096 context, head-dimension matching.}
\begin{center}
\begin{tabular}{lrrrr}
\toprule
Model & Wiki PPL & LMB PPL & LMB acc. & Avg.\ acc.\\
\midrule
DeltaNet & 20.81 & 20.57 & 37.8 & 48.55\\
DeltaProduct$_2$ & 20.54 & 19.56 & 38.3 & 49.00\\
DeltaProduct$_3$ & \textbf{20.01} & \textbf{15.56} & \textbf{42.9} & \textbf{49.95}\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item LAMBADA PPL: 24.36\% lower relative to DeltaNet.
\item LAMBADA accuracy: $+5.1$ percentage points.
\item Six-task mean: $+1.40$ percentage points.
\end{itemize}
\papersource{Paper, Table 3, PDF p. 32. All models use the extended coefficient range. PPL lower; accuracy higher.}
\end{frame}

\begin{frame}{The best update count changes with model scale}
\textbf{Six-task average accuracy, head-dimension matching.}
\begin{center}
\begin{tabular}{lrrr}
\toprule
Parameters / tokens & DeltaNet & DeltaProduct$_2$ & DeltaProduct$_3$\\
\midrule
213M / 19B & 40.23 & 40.88 & \textbf{41.01}\\
392M / 35B & 43.70 & \textbf{45.08} & 44.10\\
805M / 55B & 48.55 & 49.00 & \textbf{49.95}\\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}
\item At 392M, two updates outperform three.
\item Both parameter-matching strategies show this middle-scale pattern.
\item Individual tasks can regress even when the average improves.
\end{itemize}
\papersource{Paper, Tables 3--4, PDF p. 32. Example: 805M Table 3 Winogrande drops from 55.6 to 53.0 for DP$_3$.}
\end{frame}

\begin{frame}{Extra updates suppress loss growth beyond training length}
\centering
\includegraphics[width=0.97\linewidth,height=0.52\textheight,keepaspectratio]{figures/fig-08-length.pdf}
\begin{itemize}
\item Train at 4096 tokens; evaluate position-wise loss out to about 16K.
\item 8-head DP$_2$ and 12-head DeltaNet both have 392M parameters.
\item These are token-prediction losses on the named corpora.
\end{itemize}
\papersource{Paper, Fig. 8, PDF p. 9. 501-token moving average. Appendix Figs. 16--18 extend to 32,768 tokens.}
\end{frame}

\begin{frame}{Effective rank suggests different memory dynamics}
\begin{columns}[c]
\begin{column}{0.48\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.40\textheight,keepaspectratio]{figures/fig-09-dp3.pdf}
\end{column}
\begin{column}{0.48\textwidth}
\centering
\includegraphics[width=\linewidth,height=0.40\textheight,keepaspectratio]{figures/fig-09-dn.pdf}
\end{column}
\end{columns}
\[
\operatorname{erank}(H)=\exp\!\left(-\sum_k p_k\log p_k\right),
\qquad p_k=\frac{\sigma_k}{\sum_j\sigma_j}.
\]
DeltaNet's effective rank keeps rising beyond the training context.
The authors interpret the comparison as evidence for improved forgetting.
\papersource{Paper, selected panels from Fig. 9, PDF p. 9. Layer 20/24, four heads. Effective rank is scale invariant, not a direct information measure.}
\end{frame}

\begin{frame}{Formal-language gains are task dependent}
\textbf{Extended range; scaled accuracy; best of three runs.}
\begin{center}
\setlength{\tabcolsep}{9pt}
\begin{tabular}{lrrrr}
\toprule
Model & Parity & No brackets & Brackets & Average\\
\midrule
DeltaNet & .982 & \textbf{.915} & .281 & .726\\
DeltaProduct$_2$ & .896 & .887 & .329 & .704\\
DeltaProduct$_3$ & .932 & .736 & .330 & .666\\
DeltaProduct$_4$ & \textbf{.982} & .893 & \textbf{.342} & \textbf{.739}\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Bracketed modular arithmetic improves with every tested $n_h>1$.
\item Only DP$_4$ improves the average over the extended-range DeltaNet.
\item Training lengths: 3--40; test lengths: 40--256.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 31; Appendix C.2, pp. 29--31. The table is narrower than the accompanying all-$n_h$ claim.}
\end{frame}

\begin{frame}{Treat update count as a structural capacity parameter}
\begin{enumerate}
\item \textbf{Mechanism:} distinct sequential corrections create richer, potentially rotational transitions.
\medskip
\item \textbf{Theory:} norm-controlled factors support strong finite-state constructions; small depth may require huge width.
\medskip
\item \textbf{Evidence:} better state tracking and language-model extrapolation, with task-dependent accuracy gains.
\medskip
\item \textbf{Cost:} more projections and internal recurrence steps; implementation and parameter allocation matter.
\end{enumerate}
\papersource{Paper, Secs. 4--6 and Appendix C. Suggested main discussion length: 20--30 minutes.}
\end{frame}

\begin{frame}{Paper and discussion references}
\textbf{Julien Siems, Timur Carstensen, Arber Zela, Frank Hutter,\\
Massimiliano Pontil, and Riccardo Grazzi.}
\medskip

\emph{DeltaProduct: Improving State-Tracking in Linear RNNs via
Householder Products.}
\medskip

NeurIPS 2025. arXiv version 7, revised 22 October 2025.
\medskip

\href{https://arxiv.org/abs/2502.10297v7}{arxiv.org/abs/2502.10297v7}\\
\href{https://github.com/automl/DeltaProduct}{github.com/automl/DeltaProduct}
\medskip

Figures are reproduced from the paper/source package with figure and PDF-page locators.
Algebraic examples and qualifications are reading analysis.
\medskip

The following slides provide supporting details and corrections.
\end{frame}

\appendix

\begin{frame}{Backup: head-count matching gives a similar pattern}
\begin{center}
\begin{tabular}{lrrr}
\toprule
Parameters / tokens & DeltaNet & DP$_2$ & DP$_3$\\
\midrule
213M / 19B & 40.17 & 40.35 & \textbf{41.02}\\
392M / 35B & 43.83 & \textbf{44.98} & 44.15\\
805M / 55B & 48.22 & 49.13 & \textbf{49.40}\\
\bottomrule
\end{tabular}
\end{center}
\medskip
At 392M, Table 6 uses 12, 8, and 6 heads for $n_h=1,2,3$.
\medskip

The separate 2048-context comparison in Table 5:
\begin{itemize}
\item Matched 392M: DeltaNet 44.3 vs.\ DP$_2$ 45.5.
\item Gated: 340M DeltaNet 45.9 vs.\ 393M DP$_2$ 46.4.
\item The 15B SlimPajama rows use a different data budget.
\end{itemize}
\papersource{Paper, Tables 4--5, PDF p. 32; Table 6, p. 36. Numbers above are six-task mean accuracy.}
\end{frame}

\begin{frame}{Backup: complex eigenvalues require an angle condition}
For two factors, let $c=k_1^\top k_2$. On their two-dimensional span:
\[
\tau=2-\beta_1-\beta_2+\beta_1\beta_2c^2,\qquad
\delta=(1-\beta_1)(1-\beta_2).
\]
\[
\lambda_{\pm}=\frac{\tau\pm\sqrt{\tau^2-4\delta}}{2}.
\]
\begin{itemize}
\item Complex eigenvalues occur exactly when $\tau^2<4\delta$.
\item $\beta_1>1$ and $\beta_2>1$ are necessary, not sufficient alone.
\item Equal keys collapse to one effective factor.
\item Orthogonal keys give commuting factors and a symmetric matrix.
\end{itemize}
\papersource{Paper, Proposition 1 and proof, PDF pp. 17--18. Retain the discriminant condition when reading the final ``iff'' sentence.}
\end{frame}

\begin{frame}{Backup: spectral radius alone does not ensure stability}
Appendix B.6 states an overly strong equivalence. A counterexample is
\[
J=\begin{pmatrix}1&1\\0&1\end{pmatrix},
\qquad
J^t=\begin{pmatrix}1&t\\0&1\end{pmatrix}.
\]
Every $\rho(J^t)=1$, but $\norm{J^t}$ grows without bound.
\medskip

\textbf{DeltaProduct's valid argument uses the stronger bound}
\[
\norm{A_i}\le1 \quad\Rightarrow\quad \norm{A_t\cdots A_1}\le1.
\]
The RWKV-7 example in the paper has
$\rho(AA')=(27+\sqrt{153})/32\approx1.2303$.
It establishes an allowed unstable configuration.
\papersource{Paper, Appendix B.6 and Theorem 8, PDF pp. 27--28. Jordan example is a correction; the printed RWKV matrices swap the initially named keys.}
\end{frame}

\begin{frame}{Backup: geometric and measurement qualifications}
\begin{itemize}
\item \textbf{Dodecahedron:} its rotational symmetry group is $A_5$;
its full symmetry group is $A_5\times C_2$, not $S_5$.
\medskip
\item \textbf{Why distinguish them?} Central inversion commutes with all
symmetries, whereas $S_5$ has a trivial center.
\medskip
\item \textbf{Effective rank:} for nonzero $H$, $1\le\operatorname{erank}(H)$.
Low rank does not by itself mean a zero or empty memory.
\medskip
\item \textbf{Factor-range labels:} $[-1,1]$ labels the individual factors;
the full product can have complex eigenvalues in the unit disk.
\end{itemize}
\papersource{Corrections to Appendix C.1, PDF p. 29, and Secs. 4--5.3, pp. 5--9; supported by the stated algebra and definitions.}
\end{frame}

\begin{frame}{Backup: the 805M matched comparison extends to 32K}
\centering
\begingroup
\footnotesize
\setlength{\tabcolsep}{0pt}
\hspace*{0.04\linewidth}%
\begin{tabular}{@{}*{4}{>{\centering\arraybackslash}p{0.215\linewidth}}@{}}
CodeParrot & OpenThoughts Math & TriviaQA & SlimPajama-6B
\end{tabular}
\endgroup
\par\smallskip
\includegraphics[width=0.99\linewidth,height=0.49\textheight,keepaspectratio]{figures/fig-17-loss-805m.pdf}
\begin{itemize}
\item Same 805M scale and 55B training-token budget; head-count matching.
\item Code and math show pronounced divergence of the one-update model.
\item These are smoothed token-loss curves, not answer-success rates.
\end{itemize}
\papersource{Paper, Fig. 17 (805M row), PDF p. 35; Table 6, p. 36. Training context: 4096; maximum test context: 32,768.}
\end{frame}

\end{document}