% Self-authored reading presentation based on the arxiv-paper-reader template.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\clip}{\operatorname{Clip}_{[-c,c]}}
\newcommand{\norm}[1]{\lVert#1\rVert_2}
\title[Residual Linear Attention]{Enhancing Linear Attention\\with Residual Learning}
\subtitle{RLA and Residual Delta Net: remembering prediction errors}
\author{\texorpdfstring{Paper by Xunhao Lai, Jialiang Kang, Jianqiao Lu,\\Tong Lin, and Pengyu Zhao}{Paper by Xunhao Lai, Jialiang Kang, Jianqiao Lu, Tong Lin, and Pengyu Zhao}}
\date{arXiv:2509.25223v1 $\mid$ Research group reading $\mid$ 13 November 2026}
\begin{document}

\begin{frame}
\titlepage
\vspace{-4mm}
\begin{center}\small Reading presentation prepared with Codex.\end{center}
\end{frame}

\begin{frame}{A second recurrent state learns how the first one errs}
\begin{block}{One-sentence idea}
Fit historical prediction residuals with an auxiliary memory, then use it to correct the base memory's output.
\end{block}
\begin{itemize}
\item RLA: additive updates; RDN: delta updates for both states.
\item RDN WikiText ppl: \alert{16.57}, versus GDN's 17.27.
\item RLA NIAH: \alert{83.6\%}, versus sGLA's 76.6\%.
\end{itemize}
\medskip
{\small \href{https://aclanthology.org/2026.emnlp-main.2394/}{Published at EMNLP 2026, main proceedings}.\\
No public implementation found as of 13 November 2026.\\
This reading uses \href{https://arxiv.org/abs/2509.25223v1}{arXiv v1 (24 September 2025)} throughout.}
\papersource{Paper, Secs. 3--4, PDF pp. 4--7, Tables 2--3. Publication/code checked 2026-11-13.}
\end{frame}

\begin{frame}{Linear attention is an online key-to-value mapping}
\[
q_t,k_t\in\mathbb R^{d_k},\quad v_t,o_t\in\mathbb R^{d_v},\quad
S_t\in\mathbb R^{d_v\times d_k}
\]
\[
S_t=S_{t-1}+v_tk_t^\top,\qquad o_t=S_tq_t
\]
\begin{itemize}
\item Historical associations share a fixed-size matrix.
\item Linear scaling with sequence length; fixed recurrent state during decoding.
\item Interference and limited capacity can impair exact recall.
\end{itemize}
\begin{block}{Design question}
Can another simple memory learn the correction that the base memory needs?
\end{block}
\papersource{Paper, Secs. 1--2.1, PDF pp. 1--2. Column-vector convention follows the main text.}
\end{frame}

\begin{frame}{Prediction--correction separates history from the local write}
\[
o_t=\underbrace{S_{t-1}q_t}_{\text{base prediction}}
+\underbrace{v_t(k_t^\top q_t)}_{\text{current correction}}
\]
For Gated DeltaNet:
\[
o_t=\alpha_tS_{t-1}q_t+
\beta_t(v_t-\alpha_tS_{t-1}k_t)(k_t^\top q_t).
\]
\begin{itemize}
\item The local correction is generated by the current update.
\item Delta-rule residuals already depend on historical state.
\item RLA makes the correction itself recurrent, retaining earlier errors.
\end{itemize}
\papersource{Paper, Sec. 2.3 and Table 1, PDF p. 3. ``Single-token'' refers to the local update.}
\end{frame}

\begin{frame}{Online losses give simple, reusable state updates}
\begin{block}{Negative inner product gives additive writing}
\[
\mathcal L_{\mathrm{inner}}=-\langle Sk,v\rangle
\quad\Longrightarrow\quad S_t=S_{t-1}+v_tk_t^\top
\]
\end{block}
\begin{block}{Squared error gives the delta rule}
\[
\mathcal L_{\mathrm{inner}}=\tfrac12\norm{Sk-v}^2
\]
\[
S_t=S_{t-1}+\beta_t(v_t-S_{t-1}k_t)k_t^\top
\]
\end{block}
\begin{itemize}
\item The state acts as online parameters of a key-to-value predictor.
\item Residual fitting adds a second predictor with analogous update rules.
\end{itemize}
\papersource{Paper, Sec. 2.2, PDF pp. 2--3; App. A, PDF p. 13.}
\end{frame}

\begin{frame}{L2 prediction error provides the auxiliary target}
\[
\hat v_t=S_{t-1}k_t,\qquad r_t=v_t-\hat v_t
\]
\begin{block}{Minimal residual fitting, without gates or clipping}
\[
\begin{aligned}
R_t&=R_{t-1}+r_tk_t^\top,\\
o_t&=S_{t-1}q_t+R_tq_t,\\
S_t&=S_{t-1}+v_tk_t^\top.
\end{aligned}
\]
\end{block}
\begin{itemize}
\item $R_t$ has the same $d_v\times d_k$ shape as $S_t$.
\item The target is an error vector; the auxiliary state persists across tokens.
\item The Taylor/Newton argument motivates $r_t$; no Hessian inversion is implemented.
\end{itemize}
\papersource{Paper, Sec. 3.1, PDF pp. 3--4.}
\end{frame}

\begin{frame}{RLA decouples base learning from residual correction}
\[
\begin{aligned}
r_t&=\clip(v_t-S_{t-1}k_t),\\
R_t&=\alpha_tR_{t-1}+\gamma_tr_tk_t^\top,\\
S_t&=\alpha_tS_{t-1}+\beta_tv_tk_t^\top,\\
o_t&=\alpha_tS_{t-1}q_t+\gamma_tR_tq_t.
\end{aligned}
\]
\begin{itemize}
\item $\alpha_t$: retention; $\beta_t$: base update; $\gamma_t$: correction update/readout.
\item Residuals read the \alert{old, undecayed} $S_{t-1}$.
\item The output reads old $S_{t-1}$ and \alert{updated} $R_t$.
\item Both the write and the readout use $\gamma_t$; main experiments use $c=1$.
\end{itemize}
\papersource{Paper, Secs. 3.2--3.4, PDF pp. 4--5; App. D, Listing 1, PDF p. 15.}
\end{frame}

\begin{frame}{The correction attends to a history of errors}
For $R_0=0$, unrolling RLA gives
\[
R_t=\sum_{i=1}^t\left(\prod_{j=i+1}^t\alpha_j\right)\gamma_ir_ik_i^\top.
\]
\[
\gamma_tR_tq_t=\sum_{i=1}^t\gamma_t\gamma_i
\left(\prod_{j=i+1}^t\alpha_j\right)r_i(k_i^\top q_t).
\]
\begin{itemize}
\item Similar historical keys retrieve the errors associated with them.
\item Decay controls how long an error remains useful.
\item The current correction is scaled by $\gamma_t^2$ after write and readout.
\end{itemize}
\papersource{Teaching expansion of the RLA equations, Sec. 3.4, PDF p. 5; not a separate paper theorem.}
\end{frame}

\begin{frame}{RDN uses delta rules for both memory states}
Keep the same clipped target $r_t=\clip(v_t-S_{t-1}k_t)$:
\[
\begin{aligned}
R_t&=\alpha_tR_{t-1}(I-\gamma_tk_tk_t^\top)+\gamma_tr_tk_t^\top,\\
S_t&=\alpha_tS_{t-1}(I-\beta_tk_tk_t^\top)+\beta_tv_tk_t^\top,\\
o_t&=\alpha_tS_{t-1}q_t+\gamma_tR_tq_t.
\end{aligned}
\]
\begin{itemize}
\item Base delta update fits $v_t$; auxiliary delta update fits $r_t$.
\item Auxiliary update internally corrects $r_t-\alpha_tR_{t-1}k_t$.
\item This internal error differs from the base residual used to form $r_t$.
\item With normalized keys, the transition shrinks the current key direction.
\end{itemize}
\papersource{Paper, Sec. 3.4, PDF p. 5.}
\end{frame}

\begin{frame}{Clipping isolates a robust objective inside the target}
For scalar error $e=v-\hat v$, Huber loss is
\[
\ell_c(e)=\begin{cases}\tfrac12e^2,&|e|\le c,\\
c(|e|-\tfrac12c),&|e|>c.
\end{cases}
\]
\[
-\nabla_{\hat v}\mathcal L_{\mathrm{Huber}}
=\clip(v-\hat v).
\]
\begin{itemize}
\item Outer mapping loss defines a pseudo-residual; inner loss updates each state.
\item Nonlinearity stays in the target; state updates remain kernel-friendly.
\item Log-Cosh would yield $\tanh(v-\hat v)$; the paper proposes this without experiments.
\end{itemize}
\papersource{Paper, App. A--B, PDF pp. 13--14. App. B's two subtractive writes conflict with the main text and pseudocode; see backup.}
\end{frame}

\begin{frame}{The residual path sits inside the attention block}
\begin{center}
\includegraphics[width=.96\linewidth,height=.64\textheight,keepaspectratio]{figures/fig-01-architecture.pdf}
\end{center}
\begin{itemize}
\item Purple path: predict with old state, clip the error, and fit auxiliary memory.
\item SiLU and L2-normalized $q,k$; scalar retention/write/correction gates.
\end{itemize}
\papersource{Original Fig. 1, PDF p. 5. Gate definitions and module structure in Sec. 3.4.}
\end{frame}

\begin{frame}{Two fitting stages retain linear scaling, with more state}
\begin{enumerate}
\item Compute the base recurrence and old-state reads $S_{t-1}k_t$.
\item Form clipped residual values $r_t$.
\item Reuse a linear-attention/delta kernel to fit and read the auxiliary state.
\end{enumerate}
\medskip
\begin{itemize}
\item Base updates do not depend on $R$: the stages have an acyclic dependency.
\item Triton kernels build on FLA and return intermediate residual information.
\item Fixed dimensions: $\mathcal O(T)$ sequence work; fixed-size decoding state.
\item At the same head shape, two matrices replace one: state capacity increases.
\end{itemize}
\papersource{Paper, Sec. 4.1, PDF p. 5. Stage ordering explains the dependency; state-count comparison follows the equations.}
\end{frame}

\begin{frame}{Main results use roughly 1.5B parameters and 100B tokens}
\begin{center}
\begin{tabular}{lcc}
\toprule Property & Transformer & Linear attention \\
\midrule
Parameters & 1.51B & 1.55B \\
Hidden / intermediate & 2,048 / 8,192 & 2,048 / 8,192 \\
Heads / GQA groups & 32 / 4 & 16 / 16 \\
Head dimension & 128 & 128 \\
Layers & 16 softmax & 16 linear \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Baselines: sGLA for RLA, GDN for RDN; RetNet/Mamba2; Transformer reference.
\item Shared AdamW and cosine schedule, 4M-token batch, $c=1$.
\item Training corpus and context length are not specified in this version.
\end{itemize}
\papersource{Paper, Sec. 4.1, PDF pp. 5--6; App. C, Tables 6--7, PDF p. 15.}
\end{frame}

\begin{frame}{Kernel scaling remains linear from 2K to 128K}
\begin{center}
\includegraphics[width=.98\linewidth,height=.58\textheight,keepaspectratio]{figures/fig-02-efficiency.pdf}
\end{center}
\begin{itemize}
\item Time (left, lower is better); throughput (right, higher is better).
\item Extra fitting costs computation; long-sequence throughput stays relatively flat.
\item Hardware/batch/precision details are absent; 128K is an efficiency test length.
\end{itemize}
\papersource{Original Fig. 2, PDF p. 6. Do not infer 128K retrieval accuracy or exact speedups from this plot.}
\end{frame}

\begin{frame}{RDN leads the language-modeling and task averages}
\begin{center}
\begin{tabular}{lrrr}
\toprule Model & Wiki ppl $\downarrow$ & LAMB ppl $\downarrow$ & Eight-task acc.\ (\%) $\uparrow$ \\
\midrule
Transformer & 17.33 & 19.53 & 46.51 \\
RetNet & 18.86 & 27.62 & 43.69 \\
Mamba2 & 18.42 & 20.80 & 45.54 \\
sGLA & 17.63 & 18.06 & 45.79 \\
GDN & 17.27 & 15.76 & 46.03 \\
RLA & 17.35 & 15.59 & 46.44 \\
RDN & \textbf{16.57} & \textbf{14.93} & \textbf{47.20} \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item RLA vs.\ sGLA: +0.65 pp mean accuracy; RDN vs.\ GDN: +1.17 pp.
\item Mean mixes raw accuracy and normalized accuracy, as specified by task.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 7; 100B-token main experiments.}
\end{frame}

\begin{frame}{Aggregate gains include task-specific losses}
\begin{center}
\begin{tabular}{lrrr}
\toprule Task accuracy (\%) & GDN & RDN & Difference (pp) \\
\midrule
ARC-Easy & 54.0 & 58.7 & +4.7 \\
LAMBADA & 44.0 & 48.7 & +4.7 \\
WinoGrande & 51.9 & 49.5 & $-2.4$ \\
SocialIQA & 38.1 & 37.7 & $-0.4$ \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item RDN's overall mean exceeds the Transformer reference by 0.69 pp.
\item Transformer still scores 53.5\% on WinoGrande, versus RDN's 49.5\%.
\item Reported values are point estimates; independent training variance is not given.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 7. Differences computed from reported values.}
\end{frame}

\begin{frame}{RLA excels on NIAH; RDN has the stronger recall mean}
\begin{center}
\begin{tabular}{lrrrr}
\toprule Model & DROP & FDA & NIAH & Seven-task mean \\
\midrule
Transformer & 26.9 & 63.0 & 70.5 & 48.31 \\
sGLA & 25.7 & 51.8 & 76.6 & 45.06 \\
GDN & 25.9 & 47.4 & 75.7 & 44.99 \\
RLA & 26.5 & 51.5 & \textbf{83.6} & 46.73 \\
RDN & \textbf{27.8} & 57.5 & 79.2 & 47.11 \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Accuracy (\%); mean includes DROP/FDA/NQ/SQD/SWDE/TQA and NIAH.
\item RLA gains 7.0 pp NIAH over sGLA; RDN gains 10.1 pp FDA over GDN.
\item RDN's overall mean still trails Transformer; detailed NIAH protocol is absent.
\end{itemize}
\papersource{Paper, Table 3, PDF p. 7; task description in Sec. 4.2, PDF p. 6. Selected columns.}
\end{frame}

\begin{frame}{Persistent errors help most perplexities, with an exception}
\begin{center}
\begin{tabular}{lrrrrr}
\toprule 50B variant & Train loss & Wiki ppl & LAMB ppl & GSM8k ppl & HumanEval ppl \\
\midrule
RLA & \textbf{2.22} & \textbf{18.76} & 23.44 & \textbf{3.92} & \textbf{9.61} \\
No fitting & 2.26 & 20.19 & \textbf{22.50} & 6.85 & 16.23 \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item No fitting: replace persistent $R_t$ with $(v_t-S_{t-1}k_t)k_t^\top$.
\item All metrics are lower-is-better; LAMBADA favors the stateless variant.
\item GSM8k/HumanEval measure perplexity here, not solve rate or pass@1.
\item The table supports a broad benefit, narrower than the paragraph's wording.
\end{itemize}
\papersource{Paper, Table 4 and Sec. 4.3, PDF p. 7. 50B-token ablation, distinct from main results.}
\end{frame}

\begin{frame}{An independent correction rate lowers validation loss}
\begin{columns}[T]
\begin{column}{.54\textwidth}
\centering
\includegraphics[width=\linewidth,height=.65\textheight,keepaspectratio]{figures/fig-03a-gamma-loss.pdf}
\end{column}
\begin{column}{.44\textwidth}
\begin{itemize}
\item $x$: training tokens, 50B--100B.
\item $y$: WikiText evaluation loss; lower is better.
\item ``w/o $\gamma$'' means tying $\gamma$ to $\beta$.
\item Independent $\gamma$ improves both variants, more visibly for RDN.
\end{itemize}
\end{column}
\end{columns}
\papersource{Original Fig. 3a and Sec. 4.3, PDF p. 8. Curve values are not digitized into exact claims.}
\end{frame}

\begin{frame}{The dedicated gate's task gains vary by category}
\begin{center}
\includegraphics[width=.86\linewidth,height=.63\textheight,keepaspectratio]{figures/fig-03b-gamma-bench.pdf}
\end{center}
\begin{itemize}
\item Higher is better; groups are Reasoning, Understanding, and Recall.
\item RDN benefits across these groups; RLA Understanding shows no positive gain.
\end{itemize}
\papersource{Original Fig. 3b, PDF p. 8. Bars compare baseline, tied correction rate, and independent correction rate.}
\end{frame}

\begin{frame}{Output gates are less essential than residual memory}
\begin{center}
\begin{tabular}{llr}
\toprule 50B model & Output combination & Mean accuracy (\%) \\
\midrule
RLA & $\alpha_tS_{t-1}q_t+\gamma_tR_tq_t$ & 43.30 \\
RLA & $S_{t-1}q_t+R_tq_t$ & \textbf{43.81} \\
RDN & $\alpha_tS_{t-1}q_t+\gamma_tR_tq_t$ & \textbf{44.65} \\
RDN & $S_{t-1}q_t+R_tq_t$ & 43.96 \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Removing output gates improves RLA by 0.51 pp, reduces RDN by 0.69 pp.
\item Both variants retain auxiliary-state learning in this ablation.
\item This intervention differs from tying $\gamma=\beta$ in state learning.
\end{itemize}
\papersource{Paper, Table 5, PDF p. 8. Same 50B-token budget and hyperparameters within this ablation.}
\end{frame}

\begin{frame}{Clipping and normalization matter more for additive RLA}
\begin{columns}[T]
\begin{column}{.58\textwidth}
\centering
\includegraphics[width=\linewidth,height=.63\textheight,keepaspectratio]{figures/fig-04b-training-loss.pdf}
\end{column}
\begin{column}{.40\textwidth}
\begin{itemize}
\item RLA without normalization or clipping has higher training loss.
\item RDN and RDN without clipping nearly overlap.
\item Delta updates empirically reduce sensitivity to residual clipping.
\end{itemize}
\end{column}
\end{columns}
\papersource{Original Fig. 4b, PDF p. 9. Training loss across roughly 20B--50B tokens; activation evidence in backup.}
\end{frame}

\begin{frame}{The practical comparison needs state and compute budgets}
\begin{itemize}
\item \textbf{Capacity:} two same-shaped matrices replace one; no equal-state-budget control is reported.
\item \textbf{Cost:} extra fitting preserves linear scaling but adds constant overhead.
\item \textbf{Protocol:} corpus, training length, detailed retrieval settings, and full kernel configuration are unspecified in v1.
\item \textbf{Attribution:} compare a wider single state against two residual-fitting states at matched memory/compute.
\end{itemize}
\begin{block}{Reading assessment}
A modular memory-learning mechanism with useful gains; the best deployment choice depends on task and budget.
\end{block}
\papersource{Analysis from Secs. 3--4 and App. C--D, PDF pp. 4--9, 14--15.}
\end{frame}

\begin{frame}{Remembering errors is a useful axis for memory design}
\begin{enumerate}
\item \textbf{Mechanism:} a persistent key-to-residual map complements the base key-to-value map.
\medskip
\item \textbf{Computation:} additive or delta inner updates fit robust pseudo-residuals with reusable linear-attention kernels.
\medskip
\item \textbf{Evidence:} RDN leads aggregate language metrics; RLA leads NIAH; single-task and gating ablations reveal tradeoffs.
\end{enumerate}
\papersource{Paper, Secs. 3--6, PDF pp. 3--9. Main talk: approximately 25 minutes; backup details follow.}
\end{frame}

\begin{frame}{References and version identity}
\begin{itemize}
\item Lai et al.\ \emph{Enhancing Linear Attention with Residual Learning}.\\
\href{https://arxiv.org/abs/2509.25223v1}{arXiv:2509.25223v1}, 24 September 2025.\\
All equations, figures, tables, and PDF locators use this version.
\medskip
\item \href{https://aclanthology.org/2026.emnlp-main.2394/}{EMNLP 2026 main proceedings record}.\\
\href{https://doi.org/10.18653/2026.emnlp-main.2394}{DOI: 10.18653/2026.emnlp-main.2394}. Publication checked 13 November 2026.
\medskip
\item No paper-specific public implementation found on this lookup.\\
Appendix D provides recurrent RLA pseudocode.
\end{itemize}
\papersource{Figures use original author TeX assets. Author spelling follows arXiv metadata; v1 PDF spells Kang's given name ``Jiangliang''.}
\end{frame}

\pdfstringdefDisableCommands{\def\translate#1{#1}}
\appendix
\begin{frame}{Backup: outer targets and inner updates can be separated}
\[
r_t=-\frac{\partial\mathcal L_{\mathrm{outer}}(f(k_t;S_{t-1}),v_t)}
{\partial f(k_t;S_{t-1})}
\]
\[
\begin{aligned}
R_t&=R_{t-1}-\nabla_R\mathcal L_{\mathrm{inner}}(f(k_t;R_{t-1}),r_t),\\
S_t&=S_{t-1}-\nabla_S\mathcal L_{\mathrm{inner}}(f(k_t;S_{t-1}),v_t),\\
o_t&=f(q_t;S_{t-1})+f(q_t;R_t).
\end{aligned}
\]
\begin{itemize}
\item Gating omitted here, as in the general framework.
\item Outer mapping loss chooses the error signal; inner loss chooses additive/delta fitting.
\item This local outer loss is distinct from end-to-end language-model training loss.
\end{itemize}
\papersource{Paper, App. A, PDF p. 13. Online gradient-boosting interpretation.}
\end{frame}

\begin{frame}{Backup: update order and the appendix sign inconsistency}
\begin{enumerate}
\item Normalize $q_t,k_t$ and read the old base state to form $r_t$.
\item Update $R_t$, then combine old $S_{t-1}$ with updated $R_t$.
\item Update $S_t$ for the next token.
\end{enumerate}
\medskip
\begin{itemize}
\item Pseudocode uses row vectors and transposed states: $k_t^\top v_t$, $q_tS$.
\item Appendix B prints two \emph{subtractive} outer-product writes.
\item For $\mathcal L_{\mathrm{inner}}=-\langle Rk,r\rangle$, gradient descent gives $R_t=R_{t-1}+r_tk_t^\top$.
\item Main formulas and pseudocode use the consistent additive sign.
\end{itemize}
\papersource{Paper, Sec. 3.4, PDF p. 5; App. B, PDF p. 14; App. D, Listing 1, PDF p. 15.}
\end{frame}

\begin{frame}{Backup: missing safeguards enlarge RLA activations}
\begin{center}
\includegraphics[width=.77\linewidth,height=.62\textheight,keepaspectratio]{figures/fig-04a-activation.pdf}
\end{center}
\begin{itemize}
\item Layer index ($x$); attention-output L2 norm ($y$).
\item Without normalization/clipping, several layers reach norms of $10^2$--$10^3$.
\end{itemize}
\papersource{Original Fig. 4a, PDF p. 9. Complements the training-loss curves in the main talk.}
\end{frame}

\begin{frame}{Backup: training hyperparameters and state accounting}
\begin{center}
\begin{tabular}{lr}
\toprule Property & Value \\
\midrule
Peak / minimum learning rate & $3\times10^{-4}$ / $3\times10^{-5}$ \\
Batch / warmup / total tokens & 4M / 0.5B / 100B \\
Weight decay / gradient clipping & 0.1 / 1.0 \\
Initialization standard deviation & 0.006 \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item 16 heads, 128-dimensional keys/values: 262,144 elements per state per layer.
\item Two states: 524,288 elements per layer, derived from the reported shape.
\item These element counts do not assume a storage dtype or measure peak memory.
\end{itemize}
\papersource{Paper, App. C, Tables 6--7, PDF p. 15. State counts calculated from model dimensions.}
\end{frame}
\end{document}