% Self-authored discussion deck based on the supplied skill template.
% Build: python3 build_slides.py slides.en.tex --pdflatex /path/to/pdflatex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\usepackage{lmodern}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\diag}{\operatorname{Diag}}
\newcommand{\R}{\mathbb R}
\title[Linear attention]{Linear Attention Architectures}
\subtitle{Mechanisms, Trade-offs, and Cross-Layer Routing}
\author{Paper: Tommaso Cerruti, Tim Rieder, George Rowlands,\\Lingfeng Jin, and Imanol Schlag}
\institute{ETH Zurich\\Independent paper discussion prepared with Codex}
\date{arXiv:2607.07953v1 \quad$\cdot$\quad 8 September 2026}

\begin{document}
\begin{frame}
\titlepage
\centering
\alert{Memory control, training trade-offs, and a small gain from routing values.}
\papersource{\href{https://arxiv.org/abs/2607.07953v1}{Versioned paper}; submitted 8 July 2026. Main discussion: about 25--30 minutes, then backup material.}
\end{frame}

\begin{frame}{The paper yields three different answers}
\begin{block}{Best validation loss in the reported 350M / 15B sweep}
Kimi Delta Attention + Muon + hybrid: \textbf{2.273}.
\end{block}
\begin{block}{Fastest normalized training throughput in that sweep}
Gated DeltaNet + AdamW + pure: \textbf{100\%}, final loss \textbf{2.433}.
\end{block}
\begin{block}{Cross-layer routing}
CLVR lowers loss in five matched single runs, with \textbf{small gains}.
\end{block}
\papersource{Paper, Table 2, PDF p.12; Table 7, p.16. Training speed only; no empirical inference benchmark.}
\end{frame}

\begin{frame}{Fixed memory trades explicit retrieval for compression}
\begin{columns}[T]
\column{0.49\textwidth}
\begin{block}{Softmax attention}
Each query scores previous keys and forms a normalized value mixture.
\end{block}
\begin{itemize}
\item Explicit token-to-token retrieval.
\item Pairwise computation grows as $T^2$.
\end{itemize}
\column{0.49\textwidth}
\begin{block}{Recurrent linear attention}
History is compressed into a fixed-size associative memory.
\end{block}
\begin{itemize}
\item Read/write cost per token is independent of $T$ at fixed dimensions.
\item Correlated keys can interfere.
\end{itemize}
\end{columns}
\medskip
\alert{Two questions: how should memory be edited, and what should cross depth?}
\papersource{Paper, Secs.1--2, PDF pp.1--4. $T$: sequence length. Explicit matrix storage has quadratic memory cost; this is not a claim about every softmax implementation.}
\end{frame}

\begin{frame}{The memory maps key features into value space}
\[
\kappa^{(i)}=\phi(k^{(i)})\in\R^{d_\phi},\qquad
W^{(i)}\in\R^{d_v\times d_\phi}
\]
\[
\underbrace{\bar v^{(i)}=W^{(i-1)}\kappa^{(i)}}_{\text{predict at current key}}
\qquad
\underbrace{y^{(i)}=W^{(i)}\phi(q^{(i)})}_{\text{output at current query}}
\]
\begin{itemize}
\item $v^{(i)}\in\R^{d_v}$: current value target; $W$: dynamic state.
\item Additive memory: $W^{(i)}=W^{(i-1)}+v^{(i)}\kappa^{(i)\top}$.
\item Old-state key read and new-state query read have different roles.
\end{itemize}
\papersource{Paper, Eqs.4--10, PDF pp.3--4. The kernel discussion uses an unnormalized read. We follow the consistent matrix orientation in Eqs.5/8; see backup note on Eq.3.}
\end{frame}

\begin{frame}{DeltaNet writes a correction to the current association}
\[
r^{(i)}=v^{(i)}-W^{(i-1)}\kappa^{(i)}
\]
\[
\boxed{W^{(i)}=W^{(i-1)}+\beta^{(i)}r^{(i)}\kappa^{(i)\top}}
\]
\begin{itemize}
\item $\beta^{(i)}\in(0,1)$ controls the delta update.
\item A well-predicted association produces a small write.
\item There is no separate gate for globally decaying stale state.
\end{itemize}
\begin{block}{Teaching example: one unit-norm key}
Prediction $2$, target $5$, $\beta=0.5$ $\Rightarrow$ new prediction $3.5$.
\end{block}
\papersource{Paper, Sec.3.2, Eq.14, PDF p.5. Numerical example is illustrative, not an experiment.}
\end{frame}

\begin{frame}{GDN and KDA change the granularity of forgetting}
\begin{block}{Gated DeltaNet: one scalar for the whole state}
\[
r_\alpha=v-\alpha W_{\rm old}\kappa,\qquad
W_{\rm new}=\alpha W_{\rm old}+\beta r_\alpha\kappa^\top
\]
\end{block}
\begin{block}{Kimi Delta Attention: one decay per key-feature channel}
\[
D_\alpha=\diag(\boldsymbol\alpha),\quad
r_{\boldsymbol\alpha}=v-W_{\rm old}D_\alpha\kappa
\]
\[
W_{\rm new}=W_{\rm old}D_\alpha+\beta r_{\boldsymbol\alpha}\kappa^\top
\]
\end{block}
$D_\alpha$ multiplies on the \textbf{right}. Both mechanisms still share one scalar $\beta$ for active erase/write control.
\papersource{Paper, Eqs.15--20, PDF pp.5--6. Token superscripts suppressed. Setting $\boldsymbol\alpha=\alpha\mathbf1$ recovers GDN.}
\end{frame}

\begin{frame}{GDN2 separates key-side erase and value-side write}
\[
\widetilde W=W_{\rm old}D_\alpha,\qquad
e=\mathbf b\odot\kappa,\qquad z=\mathbf w\odot v
\]
\[
\boxed{W_{\rm new}=\widetilde W+
\underbrace{(z-\widetilde W e)}_{r_{\rm GDN2}}\kappa^\top}
\]
\begin{itemize}
\item $\mathbf b\in(0,1)^{d_\phi}$ controls the erase direction.
\item $\mathbf w\in(0,1)^{d_v}$ controls the write target.
\item The outer product still uses $\kappa^\top$, not $e^\top$.
\item $\mathbf b=\beta\mathbf1,\;\mathbf w=\beta\mathbf1$ recovers KDA.
\end{itemize}
\papersource{Paper, Sec.3.5, Eqs.21--24, PDF p.6. Greater control does not guarantee better finite-budget loss or throughput.}
\end{frame}

\begin{frame}{CLER sends a local write error into another value space}
\begin{columns}[c]
\column{0.62\textwidth}
\includegraphics[width=\linewidth,height=0.56\textheight,keepaspectratio]{figures/fig-01.pdf}
\column{0.36\textwidth}
\[
\widetilde v_{l,t}=v_{l,t}
+\Gamma_l r_{p(l),t}
\]
\begin{itemize}
\item $p(l)$: nearest lower routing layer.
\item Scalar $\Gamma_l$; raw residual in main tests.
\item Softmax layers pass the side-channel.
\end{itemize}
\end{columns}
\medskip
\alert{Author hypothesis: independently learned value spaces may be misaligned.}
\papersource{Paper, Fig.1, PDF p.8; Eqs.27--28, p.7. Original figure crop. Space mismatch is an interpretation, not independently established causality.}
\end{frame}

\begin{frame}{CLVR projects the write value into the residual stream}
\begin{columns}[c]
\column{0.59\textwidth}
\includegraphics[width=\linewidth,height=0.62\textheight,keepaspectratio]{figures/fig-02.pdf}
\column{0.39\textwidth}
\[
\varepsilon_{l,t}=P_l v_{l,t}
\]
\[
h_{l,t}\leftarrow h_{l,t}+\varepsilon_{l,t}
\]
\begin{itemize}
\item $P_l:\R^{d_v}\to\R^{d_{\rm model}}$.
\item Zero initial contribution.
\item Later layers and output head use the ordinary residual path.
\end{itemize}
\end{columns}
\papersource{Paper, Fig.2, PDF p.8; Eq.29 and Sec.4.3, pp.7--9. Original crop. Adds a projection; empirical routing overhead is not tabulated.}
\end{frame}

\begin{frame}{The cleanest ablation compares value with error}
\centering
\begin{tabular}{llll}
\toprule
Variant & Signal & Destination & Mapping\\
\midrule
CLER & $r$ & Later value target & Scalar $\Gamma_l$\\
CLER-H & $r$ & Residual stream & Projection $P_l$\\
CLVR & $v$ & Residual stream & Projection $P_l$\\
\bottomrule
\end{tabular}
\[
v_{l,t}=r_{l,t}+\bar v_{l,t}
\]
\begin{itemize}
\item CLER-H and CLVR use \textbf{identical projection shapes/ranks}.
\item Value retains the memory prediction subtracted from the error.
\item CLER $\to$ CLER-H also changes parameterization; it is a less isolated test of injection space.
\end{itemize}
\papersource{Paper, Secs.4.2--4.3, PDF pp.7--9; last bullet is reading analysis. Cross-table baselines also differ.}
\end{frame}

\begin{frame}{A shared recipe still contains distinct experiment groups}
\begin{itemize}
\item \textbf{Main sweep:} 350M class / 15B tokens; FineWeb-Edu, LLaMA2 tokenizer, length 4096, global batch 128, bf16, 4 GH200 GPUs.
\item \textbf{Architecture-specific depths:} 20--24 layers; recorded counts near 312M, approximately $\pm2\%$.
\item \textbf{Routing:} matched host, optimizer, schedule, seed; 1B/15B tokens at 350M, plus GDN at 1.3B/40B.
\item \textbf{Critical boundary:} different data slices across routing budgets; compare gains within matched groups.
\end{itemize}
\begin{block}{Metric and statistical scope}
Final held-out cross-entropy, not the best intermediate point.\\
Single runs: no seed-averaged estimates or standard deviations.
\end{block}
\papersource{Paper, Sec.5, Table 1, PDF pp.9--11; Sec.6, p.11; Appendix B, p.20.}
\end{frame}

\begin{frame}{KDA achieves lower loss at lower training throughput}
\centering
\begin{tabular}{llrr}
\toprule
Configuration & Stack & Final loss $\downarrow$ & Throughput $\uparrow$\\
\midrule
Softmax / Muon & dense & 2.349 & 79.7\%\\
DeltaNet / Muon & hybrid & 2.299 & 83.8\%\\
GDN / Muon & hybrid & 2.321 & 89.5\%\\
KDA / Muon & hybrid & \textbf{2.273} & 70.9\%\\
GDN2 / Muon & hybrid & 2.345 & 81.3\%\\
GDN / AdamW & pure & 2.433 & \textbf{100.0\%}\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Versus softmax/Muon: KDA hybrid lowers loss by 0.076, with about 11.0\% lower throughput (calculated).
\item Hybrid loss is lower within each pair; throughput need not fall: KDA/Muon hybrid 70.9\% vs.\ pure 68.5\%.
\end{itemize}
\papersource{Paper, Table 2, PDF p.12; 350M / 15B, single runs. Hybrid: linear + softmax; pure: linear mixers only. Throughput normalized to GDN/AdamW/pure.}
\end{frame}

\begin{frame}{Optimizer effects must be read with learning-rate choice}
\begin{columns}[c]
\column{0.61\textwidth}
\includegraphics[width=\linewidth,height=0.66\textheight,keepaspectratio]{figures/fig-05.pdf}
\column{0.37\textwidth}
\begin{itemize}
\item 350M hybrid, 2000 steps ($\sim$1.05B tokens).
\item Solid: Muon; dashed: AdamW.
\item Muon favors roughly $3\times10^{-4}$.
\item Poor learning rates can reverse a comparison.
\end{itemize}
\end{columns}
\papersource{Paper, Fig.5, PDF p.13 (original crop), Sec.6.2, p.12. Sec.5.2 and Sec.6.2 differ in how broadly they assign AdamW's optimum to $10^{-3}$; treat the individual curves as the evidence.}
\end{frame}

\begin{frame}{Pure GDN has flatter measured training iteration times}
\begin{columns}[c]
\column{0.60\textwidth}
\includegraphics[width=\linewidth,height=0.61\textheight,keepaspectratio]{figures/fig-06.pdf}
\column{0.38\textwidth}
\textbf{At 32k tokens:}
\begin{itemize}
\item Softmax: 3.37 s/iteration.
\item GDN hybrid: 1.56 s.
\item GDN pure: 0.96 s.
\end{itemize}
Softmax/pure time ratio: \textbf{3.51} (calculated).
\end{columns}
\medskip
Per-length token/batch control is not fully specified in the text.\\
\alert{This is a training measurement; inference speed is unmeasured.}
\papersource{Paper, Fig.6, PDF p.14, Sec.6.3, p.12. Original crop. Error bars show step-to-step variation, not training-seed uncertainty.}
\end{frame}

\begin{frame}{Larger DeltaNet runs remain learning-rate sensitive}
\centering
\begin{tabular}{llrr}
\toprule
Params / tokens & Configuration & Learning rate & Final loss\\
\midrule
1.3B / 40B & DN hybrid & $3.6\times10^{-4}$ & 2.066\\
1.3B / 40B & DN pure & $1.5\times10^{-4}$ & 2.063\\
1.3B / 40B & DN pure + CLER & $1.5\times10^{-4}$ & 2.112\\
3B / 60B & DN hybrid & $5\times10^{-4}$ & 2.332\\
3B / 60B & DN hybrid & $3\times10^{-4}$ & 1.955\\
3B / 60B & DN hybrid & $1.5\times10^{-4}$ & 1.955\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item A larger-scale DeltaNet study, not an all-architecture sweep.
\item The CLER row is not a CLVR result.
\end{itemize}
\papersource{Paper, Table 3, PDF p.14; selected rows. Optimizer is not listed row-by-row in this table.}
\end{frame}

\begin{frame}{Original CLER produces no reliable validation gain}
\centering
\begin{tabular}{llrrr}
\toprule
Optimizer & Host & Baseline & CLER & Reported $\Delta$\\
\midrule
Muon & GDN & 2.8319 & 2.8333 & $+0.0013^{*}$\\
Muon & DeltaNet & 2.8511 & 2.8507 & $-0.0004$\\
AdamW & GDN & 3.2554 & 3.2562 & $+0.0008$\\
AdamW & DeltaNet & 3.4068 & 3.4258 & $+0.0190$\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item 350M / approximately 1B tokens; matched per host and optimizer.
\item The only favorable difference is below $10^{-3}$ in one run.
\item Nonzero routed residuals do not establish a useful causal effect.
\end{itemize}
\papersource{Paper, Table 6, Sec.6.5, PDF p.15. Negative $\Delta$ is better. $^{*}$Paper reports +0.0013; displayed losses subtract to +0.0014. Retained as a rounding/reporting discrepancy.}
\end{frame}

\begin{frame}{CLVR improves all five matched rows by small amounts}
\centering
\begin{tabular}{llrrr}
\toprule
Host & Params / tokens & Baseline & CLER-H $\Delta$ & CLVR $\Delta$\\
\midrule
GDN & 350M / 1B & 2.8331 & $-0.0073$ & $\mathbf{-0.0103}$\\
GDN & 350M / 15B & 2.3417 & $-0.0042$ & $\mathbf{-0.0059}$\\
GDN & 1.3B / 40B & 2.0635 & $-0.0010$ & $\mathbf{-0.0019}$\\
DeltaNet & 350M / 1B & 2.8469 & $-0.0047$ & $\mathbf{-0.0119}$\\
DeltaNet & 350M / 15B & 2.3347 & $-0.0002$ & $\mathbf{-0.0016}$\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Muon; final validation loss; negative $\Delta$ is better.
\item Value beats error at equal projection shape; gains diminish in the longer/larger reported runs.
\item Single matched runs; no larger DeltaNet hidden-stream row.
\end{itemize}
\papersource{Paper, Table 7, PDF p.16. Compare within each row. Do not apply these deltas to Table 2 baselines; data slices/run groups differ.}
\end{frame}

\begin{frame}{Controls support a signal hypothesis, with limited evidence}
\begin{itemize}
\item \textbf{Exploratory 350M/1B group:}\\
plain CLVR 2.8152; surprise gate 2.8220; error+value 2.8194.
\item \textbf{Capacity controls, one seed:}\\
hidden-state routing $\Delta\approx-0.0014$; larger baseline $+0.0021$.
\item \textbf{Attention Residuals, 350M/15B:}\\
DeltaNet $\Delta=-0.0101$ (larger gain than CLVR); GDN $+0.0030$.
\end{itemize}
\begin{block}{Evidence boundary}
These controls do not establish statistical significance or universal superiority. The exploratory 2.8152 reference is a separate group from Table 7.
\end{block}
\papersource{Paper, Sec.6.6, PDF pp.16--17; Appendix B, p.20. Attention Residuals numbers occur in prose, not as Table 7 rows.}
\end{frame}

\begin{frame}{Downstream checks show mixed, small task differences}
\centering
\begin{tabular}{llrrr}
\toprule
Host & Variant & HS acc\_norm & PIQA acc & WG acc\\
\midrule
GDN & Baseline & 0.4131 & 0.6692 & 0.5146\\
GDN & CLVR & 0.4113 & 0.6719 & 0.5328\\
DeltaNet & Baseline & 0.4113 & 0.6600 & 0.5193\\
DeltaNet & CLVR & 0.4104 & 0.6616 & 0.5359\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Matched 350M/15B/Muon, zero-shot, full-sequence scoring.
\item HS: HellaSwag; WG: WinoGrande; higher is better.
\item GDN/CLVR WG is +1.82 percentage points (calculated), but a single checkpoint does not establish a routing effect.
\end{itemize}
\papersource{Paper, Table 5, PDF p.15; selected columns. Authors use these tasks to check for clear degradation. Table 4 uses a separate evaluation/run group.}
\end{frame}

\begin{frame}{The paper's strongest boundaries are empirical}
\begin{enumerate}
\item No repeated training seeds; small loss differences are suggestive.
\item Uneven hyperparameter coverage; architecture and tuning interact.
\item Routing tested only on DeltaNet and GDN; larger hidden-stream routing only on GDN.
\item No long-context retrieval/QA study and no inference benchmark.
\item Space mismatch is a plausible explanation; CLER and CLER-H also differ in parameterization.
\end{enumerate}
\papersource{Paper, Secs.5--8 and Appendix B, PDF pp.10--11, 17--18, 20; final item includes reading analysis from Eqs.27/29.}
\end{frame}

\begin{frame}{Useful next tests separate content, space, and scale}
\begin{itemize}
\item Repeat matched seeds and longer budgets to measure uncertainty.
\item Try KDA/GDN2 hosts; distinguish raw value, gated target, and residual.
\item Measure associative recall, passkeys, long-context QA, decoding memory and throughput.
\item \textbf{Reading proposal:} hold the projection budget fixed while changing injection space; intervene by disabling routing after training.
\end{itemize}
\begin{block}{Take-home message}
Choose memory control, stack, optimizer, and learning rate together.\\
CLVR is a simple, promising signal route with modest current evidence.
\end{block}
\papersource{Paper, Secs.8--9, PDF pp.17--18. The proposed matched-space and post-training interventions are analysis suggestions, not reported experiments.}
\end{frame}

\begin{frame}{Paper and reproducibility trail}
\textbf{Tommaso Cerruti, Tim Rieder, George Rowlands, Lingfeng Jin,\\
and Imanol Schlag.}
\medskip

\emph{Linear Attention Architectures: Mechanisms, Trade-offs,\\
and Cross-Layer Routing.}
\medskip

\href{https://arxiv.org/abs/2607.07953v1}{arXiv:2607.07953v1} \quad Submitted 8 July 2026.

\medskip
\href{https://github.com/tommasocerruti/linear-attention-architectures}{Author-linked implementation and experiment records}

\medskip
This reading covers the 20-page paper and Appendices A/B.\\
It does not reproduce training or audit the repository.

\papersource{Paper, Sec.6.7, PDF p.17. All displayed paper figures are original PDF crops; page locators refer to the paper, not this deck.}
\end{frame}

\appendix
\begin{frame}{Backup: the delta rule is a local gradient step}
For fixed $\kappa$ and $v$, consider the teaching objective
\[
\ell(W)=\tfrac12\|W\kappa-v\|_2^2,\qquad
\nabla_W\ell=(W\kappa-v)\kappa^\top.
\]
Then $W_{\rm new}=W_{\rm old}-\beta\nabla_W\ell$ is Eq.14.
\[
\|\kappa\|_2=1
\quad\Longrightarrow\quad
W_{\rm new}\kappa=(1-\beta)W_{\rm old}\kappa+\beta v.
\]
For another direction $u$:
\[
(W_{\rm new}-W_{\rm old})u
=\beta(v-W_{\rm old}\kappa)(\kappa^\top u).
\]
\papersource{Teaching derivation from Paper Eq.14, PDF p.5. The unit-norm assumption is explicit; this local objective is not the outer language-model training loss.}
\end{frame}

\begin{frame}{Backup: full AdamW architecture rows}
\centering
\begin{tabular}{llrrr}
\toprule
Architecture & Stack & Speed & Saturation & Final\\
\midrule
Softmax & dense & 81.7\% & 2.587 & 2.413 \\
DeltaNet & hybrid & 87.0\% & 2.517 & 2.348 \\
DeltaNet & pure & 91.9\% & 2.555 & 2.382 \\
GDN & hybrid & 90.5\% & 2.517 & 2.356 \\
GDN & pure & 100.0\% & 2.601 & 2.433 \\
KDA & hybrid & 74.4\% & 2.471 & 2.328 \\
KDA & pure & 73.6\% & 2.490 & 2.347 \\
GDN2 & hybrid & 83.3\% & 2.520 & 2.379 \\
GDN2 & pure & 81.9\% & 2.601 & 2.452 \\
\bottomrule
\end{tabular}
\medskip

350M class / 15B tokens. Speed: normalized training throughput.\\
Lower loss and higher speed are better; each row is one run.
\papersource{Paper, Table 2, PDF p.12. DN = DeltaNet, GDN = Gated DeltaNet, KDA = Kimi Delta Attention, GDN2 = Gated DeltaNet-2.}
\end{frame}
\begin{frame}{Backup: full Muon architecture rows}
\centering
\begin{tabular}{llrrr}
\toprule
Architecture & Stack & Speed & Saturation & Final\\
\midrule
Softmax & dense & 79.7\% & 2.489 & 2.349 \\
DeltaNet & hybrid & 83.8\% & 2.440 & 2.299 \\
DeltaNet & pure & 89.2\% & 2.476 & 2.334 \\
GDN & hybrid & 89.5\% & 2.464 & 2.321 \\
GDN & pure & 95.0\% & 2.543 & 2.397 \\
KDA & hybrid & 70.9\% & 2.409 & 2.273 \\
KDA & pure & 68.5\% & 2.427 & 2.290 \\
GDN2 & hybrid & 81.3\% & 2.466 & 2.345 \\
GDN2 & pure & 80.7\% & 2.542 & 2.415 \\
\bottomrule
\end{tabular}
\medskip

350M class / 15B tokens. Speed: normalized training throughput.\\
Lower loss and higher speed are better; each row is one run.
\papersource{Paper, Table 2, PDF p.12. DN = DeltaNet, GDN = Gated DeltaNet, KDA = Kimi Delta Attention, GDN2 = Gated DeltaNet-2.}
\end{frame}

\begin{frame}{Backup: early 1B baselines are a separate comparison}
\centering
\begin{tabular}{llrrr}
\toprule
Architecture & Optimizer & Val loss & Val PPL & ktok/s/GPU\\
\midrule
Softmax & AdamW & 2.9019 & 18.2095 & 144.8\\
Softmax & Muon & 2.7240 & 15.2410 & 136.7\\
GDN & AdamW & 2.8562 & 17.3950 & 159.5\\
GDN & Muon & 2.7097 & 15.0254 & 154.8\\
DeltaNet & AdamW & 2.9391 & 18.8987 & 153.4\\
DeltaNet & Muon & 2.7137 & 15.0847 & 146.7\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Earlier 350M / approximately 1B-token runs.
\item Useful optimizer context, not the final 15B frontier.
\item Do not use these losses as Table 6 or Table 7 baselines.
\end{itemize}
\papersource{Paper, Table 8 and Appendix A, PDF p.20. Throughput is in thousands of training tokens per second per GPU, without normalization.}
\end{frame}

\begin{frame}{Backup: projection cost and initialization}
\[
P_l\in\R^{d_{\rm model}\times d_v},\qquad
P_l=U_lD_l\quad\text{(optional rank }d_p\text{)}
\]
\begin{itemize}
\item Full-rank parameter count per layer: $d_{\rm model}d_v$.
\item Low-rank count: $d_p(d_{\rm model}+d_v)$.
\item At fixed width/rank, added work is linear in sequence length.
\item A hybrid host still retains its softmax-layer costs.
\item For a factorized projection, zeroing both factors blocks ordinary gradients; zero the product with a trainable path.
\end{itemize}
\papersource{Paper, Eq.29, PDF p.7. Counts and gradient observation are teaching analysis. Exact per-run rank, added parameter counts, and measured routing overhead are not tabulated in the paper.}
\end{frame}

\begin{frame}{Backup: keep formula conventions and run groups explicit}
\begin{itemize}
\item \textbf{Eq.3, p.3:} multiplication order conflicts with $W\in\R^{d_v\times d_\phi}$. Eqs.5/8 give the consistent read $W\phi(q)$.
\item \textbf{GDN residual:} Sec.3.3 uses a decay-adjusted read; Sec.4 writes a simplified ungated expression. Verify the actual internal signal before implementation.
\item \textbf{Initial state:} zero $P_l$ means zero added route, not a zero host memory or zero network parameters.
\item \textbf{Baselines:} Table 2, Table 6, Table 7, and the exploratory controls are separate run groups.
\end{itemize}
\papersource{Reading audit of Paper Eqs.3--5, 15--17, 25--29; Tables 2, 6--7; Sec.6.6. This is not a code audit.}
\end{frame}
\end{document}