% Based on the arxiv-paper-reader Beamer template; pdfLaTeX.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\usepackage{lmodern}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\R}{\mathbb R}
\newcommand{\one}{\mathbf 1}
\newcommand{\norm}[1]{\left\|#1\right\|}
\title[Lattice]{Lattice: Learning to Efficiently\newline Compress the Memory}
\subtitle{Orthogonal writes into a fixed recurrent state}
\author{Paper authors: Mahdi Karami, Razvan Pascanu, Vahab Mirrokni}
\date{arXiv:2504.05646v2 (30 December 2025)\\Paper discussion prepared 14 September 2026}
\begin{document}
% 22 main slides for approximately 25 minutes, followed by 5 backup slides.
\begin{frame}
\titlepage
\vspace{-5mm}
\begin{center}\small
\href{https://arxiv.org/abs/2504.05646v2}{arxiv.org/abs/2504.05646v2}\quad|\quad Independent reading presentation
\end{center}
\end{frame}

\begin{frame}{The question is how to write into limited memory}
\begin{columns}[T,onlytextwidth]
\column{.48\textwidth}
\begin{block}{Softmax attention}
Keep historical keys and values.\\[2mm]
Cache grows with sequence length.\\
Sequence attention work: $O(T^2d)$.
\end{block}
\column{.48\textwidth}
\begin{block}{Fixed recurrent state}
Keep $S_t\in\R^{d\times m}$.\\[2mm]
$S_t=S_{t-1}+v_tk_t^\top$.\\
Repeated writes can interfere.
\end{block}
\end{columns}
\vspace{3mm}
\alert{Can each slot inspect its contents before accepting a write?}
\begin{itemize}
\item Low-rank structure motivates a compact learned dictionary.
\item Fixed state implies lossy compression; no lossless-cache guarantee.
\end{itemize}
\papersource{Paper, Secs. 2--3, PDF pp. 2--4. Costs refer to sequence mixing, excluding projections/MLPs.}
\end{frame}

\begin{frame}{The recurrent state is a decoder learned online}
\[
 k_t=W_kx_t\in\R^m,\quad v_t\in\R^d,\quad
 S_t=[s_{1,t},\ldots,s_{m,t}]\in\R^{d\times m}.
\]
\begin{block}{Inner loop: adapt memory to this sequence}
\[
S_t=S_{t-1}-\gamma_t\nabla_S\mathcal L(S_{t-1},v_t,k_t),
\qquad y_t=S_tq_t,\quad q_t\in\R^m.
\]
\end{block}
\begin{itemize}
\item Outer training learns projections and the rule's parameters.
\item Inner adaptation updates $S$, including during inference.
\item The target comes from the current token; no future label is needed.
\end{itemize}
\papersource{Paper, Sec. 3.1, Eqs. (3)--(4), PDF pp. 4--5; readout, p. 8.}
\end{frame}

\begin{frame}{Three objectives share a normalized dictionary}
\[
\Phi=\phi(S),\qquad \phi_i=\frac{s_i}{\norm{s_i}}.
\]
\renewcommand{\arraystretch}{1.5}
\begin{tabular}{lll}
\toprule
Variant & Compression objective & Interpretation\\
\midrule
DEC & $\frac12\norm{\Phi k_t-v_t}^2$ & Decode a key/code\\
DOT / SIM & $-\langle\Phi k_t,v_t\rangle$ & Align with the value\\
ENC & $\frac12\norm{\Phi^\top v_t-k_t}^2$ & Encode the value\\
\bottomrule
\end{tabular}
\vspace{4mm}
\begin{itemize}
\item DEC error: $e_t=\Phi k_t-v_t\in\R^d$.
\item ENC error: $e_t=\Phi^\top v_t-k_t\in\R^m$.
\end{itemize}
\papersource{Paper, Eqs. (5), (7), (24), PDF pp. 5, 19. Half-squared losses here make the omitted factor 2 explicit; it can be absorbed into the step size.}
\end{frame}

\begin{frame}{Normalizing a slot makes its gradient orthogonal}
\[
J_\phi(s_i)=\frac{1}{\norm{s_i}}
\underbrace{\left(I-\frac{s_is_i^\top}{\norm{s_i}^2}\right)}_{P(s_i):\ \text{orthogonal projector}}
\]
\begin{block}{DEC: the chain rule gives a slot-specific direction}
\[
\nabla_{s_i}\mathcal L_t=\frac{k_{i,t}}{\norm{s_i}}P(s_i)e_t,
\qquad s_i^\top\nabla_{s_i}\mathcal L_t=0.
\]
\end{block}
\begin{itemize}
\item $k_{i,t}$ controls the basic write coefficient.
\item $P(s_i)$ removes the component parallel to the current slot.
\item State dependence comes from the projection and the error.
\end{itemize}
\papersource{Paper, Sec. 3.1.1, Eqs. (5)--(9), PDF pp. 5--6. Assumes nonzero slot norms.}
\end{frame}

\begin{frame}{Orthogonality is local to each current slot}
\centering
\includegraphics[width=.96\linewidth,height=.52\textheight,keepaspectratio]{figures/fig01-geometry.pdf}
\begin{itemize}
\item (a) Decompose the incoming vector; (b) write its orthogonal part.
\item (c) The state has $m$ slots of width $d$, evolving over time $t$.
\item \alert{This does not ensure mutually orthogonal slots or lossless recall.}
\end{itemize}
\papersource{Paper, Fig. 1, PDF p. 2; exact crop. The diagram uses an abstract positive write coefficient.}
\end{frame}

\begin{frame}{Renormalization stabilizes the state and changes retention}
For $\norm{s_i}=1$ and $s_i^\top\Delta s_i=0$:
\[
\norm{s_i+\Delta s_i}^2=1+\norm{\Delta s_i}^2,\qquad
s_{i,t}=\underbrace{(1+\norm{\Delta s_i}^2)^{-1/2}}_{\beta_{i,t}}(s_i+\Delta s_i).
\]
\begin{itemize}
\item Normalization \emph{inside the loss} determines the write direction.
\item Normalization \emph{after the update} controls the state norm.
\item Since $\beta\leq1$, the old direction loses relative weight.
\end{itemize}
\begin{block}{Geometric interpretation}
A tangent-space gradient step followed by a sphere retraction.\\
This is not a theorem about optimal historical compression.
\end{block}
\papersource{Paper, Sec. 3.2, Eq. (10), Proposition 3.1, PDF p. 6; proof p. 21. Formula shown without explicit decay ($\mu=1$).}
\end{frame}

\begin{frame}{The orthogonal write costs $O(dm)$ per token}
No $d\times d$ projector needs to be materialized:
\[
P(s_i)h_t=h_t-s_i\frac{s_i^\top h_t}{\norm{s_i}^2}.
\]
For unit slots, define $\hat h_t=S_{t-1}^\top h_t$ and
$\hat k_t=\gamma_t\beta_t\odot k_t$:
\[
\begin{split}
S_t={}&(\mu_t\one\beta_t^\top)\odot S_{t-1}\\
 &+[\one(\hat h_t\odot\hat k_t)^\top]\odot S_{t-1}
-h_t\hat k_t^\top.
\end{split}
\]
\begin{itemize}
\item State: $O(dm)$ per head; sequence mixing work: $O(Tdm)$.
\item DEC: $h_t=\Phi k_t-v_t$; DOT: $h_t=-v_t$.
\item Explicit L2 state decay gives $\mu_t=1-\gamma_t\lambda_t$.
\end{itemize}
\papersource{Paper, Eqs. (11), (14), PDF pp. 7--8. Fixed $d,m$; excludes the rest of the network and training activations.}
\end{frame}

\begin{frame}{Lattice is embedded in a full language model}
\begin{columns}[c,onlytextwidth]
\column{.59\textwidth}
\includegraphics[width=\linewidth,height=.73\textheight,keepaspectratio]{figures/fig04-architecture.pdf}
\column{.38\textwidth}
\begin{itemize}
\item Multi-head recurrent mixing.
\item Short $q/k$ convolutions, width 4.
\item Output normalization and GeLU post-gate.
\item Learned inner step size:\\[1mm]
$\gamma_t=\operatorname{sigmoid}(W_\gamma x_t)$.
\end{itemize}
\end{columns}
\papersource{Paper, Fig. 4 and Appendix C, PDF p. 22; exact crop. The outer block also contains RMSNorm, SwiGLU and residual connections.}
\end{frame}

\begin{frame}{Chunk parallelism requires an approximation}
Let $S_0^b$ be the state before a chunk of $C$ tokens.
\begin{enumerate}
\item Compute state-dependent quantities using $S_0^b$.
\item Evaluate the resulting recurrence in parallel within the chunk.
\item Pass the final state to the next chunk.
\end{enumerate}
\begin{block}{Simplified approximation: Eq. (17)}
\[
S_t=\hat G_t\odot S_{t-1}-h_t\hat k_t^\top,
\qquad
\tilde h_t=(S_0^b)^\top h_t,
\]
\[
\hat G_t=\one(\mu_t\beta_t+\tilde h_t\odot\hat k_t)^\top.
\]
\end{block}
\alert{Larger $C$ improves parallelism, but changes the nonlinear update.}
\papersource{Paper, Eqs. (15)--(17), PDF p. 8; Appendix B.1, pp. 18--19. DEC errors also use the chunk-start state. Inter-chunk dependencies remain.}
\end{frame}

\begin{frame}{The evaluation uses different operating points}
\renewcommand{\arraystretch}{1.45}
\begin{tabular}{lll}
\toprule
Question & Setup & Key condition\\
\midrule
Context scaling & Books3, 110M & 512--16k; $C=1$\\
Model scaling & FineWeb-Edu & 110M / 340M / 760M\\
Zero-shot tasks & 340M / 760M & 4k training; usually $C=4$\\
Component ablation & FineWeb-Edu, 110M & Usually $C=1$\\
MQAR & Two blocks, one head & $m=d$; synthetic recall\\
Training throughput & 110M; $2\times2$ TPU & $C=16$ and $C=64$\\
\bottomrule
\end{tabular}
\vspace{4mm}
\begin{itemize}
\item LM batch: 0.5M tokens. Training budgets: 5B / 15B / 30B tokens.
\item Lower perplexity is better; higher accuracy/throughput is better.
\end{itemize}
\papersource{Paper, Sec. 4 and Appendix C, PDF pp. 9--10, 22--26. MQAR uses its own batch sizes.}
\end{frame}

\begin{frame}{Long-context gains are not monotonic improvements}
\begin{columns}[c,onlytextwidth]
\column{.58\textwidth}
\includegraphics[width=\linewidth,height=.63\textheight,keepaspectratio]{figures/fig02-books.pdf}
\column{.39\textwidth}
\textbf{110M, Books, $C=1$}
\begin{itemize}
\item 8k: DEC 16.62 vs. Gated-DeltaNet 17.40.
\item 16k: DOT 16.82 vs. Gated-DeltaNet 17.49.
\item DEC worsens from 16.62 at 8k to 16.97 at 16k.
\end{itemize}
\end{columns}
\vspace{2mm}
Each context length is a separate training setting. Transformer++ has no 16k result here.
\papersource{Paper, Fig. 2 left / Fig. 5 left, PDF pp. 9, 23; exact crop. Values from the authors' plotted data in books\_plot.tex.}
\end{frame}

\begin{frame}{Perplexity improves at the three tested model scales}
\centering
\renewcommand{\arraystretch}{1.35}
\begin{tabular}{lrrr}
\toprule
Model & 110M / 5B & 340M / 15B & 760M / 30B\\
\midrule
Transformer++ & 16.09 & 11.92 & 9.75\\
Gated-DeltaNet & 15.378 & 11.438 & 9.5378\\
TTT & 15.596 & 11.81 & 9.7468\\
Lattice-DEC & \textbf{15.021} & \textbf{11.339} & \textbf{9.47}\\
Lattice-DOT & 15.033 & 11.377 & \textbf{9.47}\\
\bottomrule
\end{tabular}
\vspace{4mm}
\begin{itemize}
\item FineWeb-Edu; perplexity $\downarrow$; parameters and training tokens both grow.
\item The original horizontal axis is FLOPs, not parameter count.
\item Largest-scale DEC gain over Gated-DeltaNet: 0.0678 ppl.
\end{itemize}
\papersource{Paper, Fig. 2 right, PDF p. 9; values from Scalling\_pattern.tex. FLOPs differ slightly; three points do not establish a general scaling law.}
\end{frame}

\begin{frame}{The best variant improves average zero-shot accuracy}
\centering
\renewcommand{\arraystretch}{1.3}
\begin{tabular}{lrr}
\toprule
Model & 340M / 15B & 760M / 30B\\
\midrule
Transformer++ & 46.93 & 51.94\\
GLA & 48.68 & 52.57\\
Gated-DeltaNet & 48.60 & 52.64\\
TTT & 47.88 & 52.42\\
Lattice-DEC, $C=4$ & \textbf{49.14} & 53.54\\
Lattice-DOT, $C=4$ & 48.66 & \textbf{54.03}\\
\bottomrule
\end{tabular}
\vspace{3mm}
\begin{itemize}
\item Over the best baseline: \textbf{+0.46 pp} at 340M; \textbf{+1.39 pp} at 760M.
\item Gated-DeltaNet wins ARC-e; Transformer++ wins BoolQ at 760M.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 10. Mean accuracy (\%) over seven tasks; HellaSwag/ARC-c use normalized accuracy. No confidence intervals reported.}
\end{frame}

\begin{frame}{Orthogonal writes and normalization drive the ablation}
\centering
\renewcommand{\arraystretch}{1.35}
\begin{tabular}{lrr}
\toprule
Configuration & Perplexity $\downarrow$ & Increment\\
\midrule
DeltaNet & 16.35 & ---\\
+ Orthogonal recurrence & 15.38 & $-0.97$\\
+ State projection to the sphere & 15.07 & $-0.31$\\
+ Explicit forget gate & \textbf{15.02} & $-0.05$\\
\midrule
TTT, $C=1$ (separate baseline) & 15.60 & ---\\
Lattice, $C=4$, Eq. (15) & 15.15 & ---\\
Lattice, $C=4$, Eq. (17) & 15.12 & ---\\
\bottomrule
\end{tabular}\par
\vspace{4mm}
The additions are cumulative; this is not a full factorial experiment.
\papersource{Paper, Table 3, PDF p. 10. FineWeb-Edu, 110M, $m=d=64$; cumulative rows use $C=1$.}
\end{frame}

\begin{frame}{A quarter of the state can match a larger TTT memory}
\begin{columns}[c,onlytextwidth]
\column{.52\textwidth}
\includegraphics[width=\linewidth,height=.61\textheight,keepaspectratio]{figures/fig06-memory.pdf}
\column{.45\textwidth}
\begin{block}{110M, FineWeb-Edu, $C=1$}
Lattice, $m=16$: \textbf{15.52 ppl}.\\
TTT, $m=64$: \textbf{15.60 ppl}.
\end{block}
For $d=64$, state elements per head:
\[
64\times16=1024,\quad64\times64=4096.
\]
\end{columns}
\vspace{3mm}
\begin{itemize}
\item The $4\times$ claim concerns recurrent matrix state in this setting.
\item Changing $m$ also changes key/query projection parameters.
\end{itemize}
\papersource{Paper, Table 3 / Fig. 6, PDF pp. 10, 23; exact crop. Not a measurement of total training or inference memory.}
\end{frame}

\begin{frame}{Larger chunks expose normalization and quality costs}
\centering
\renewcommand{\arraystretch}{1.55}
\begin{tabular}{lrr}
\toprule
Eq. (15) approximation & Per-token norm & Per-chunk norm\\
\midrule
$C=4$ & 15.15 & 15.17\\
$C=16$ & 15.32 & 15.36\\
\bottomrule
\end{tabular}
\vspace{5mm}
\begin{itemize}
\item Both normalization placement and chunk size affect perplexity.
\item Separately, Eq. (17) with $C=16$ and \emph{per-chunk} normalization was reported unstable.
\item That instability is distinct from the finite values above.
\end{itemize}
\papersource{Paper, Table 5 and accompanying text, PDF p. 24. FineWeb-Edu, 110M; perplexity $\downarrow$.}
\end{frame}

\begin{frame}{Dense associative recall is stronger at limited width}
\textbf{MQAR: 256 key--value pairs, $d=m=64$, two blocks.}
\begin{center}
\renewcommand{\arraystretch}{1.3}
\begin{tabular}{lrr}
\toprule
Model & $L=1024$ & $L=2048$\\
\midrule
Transformer++ (external result) & 100 & 100\\
xLSTM[1:1] (external result) & 73 & 66\\
TTT & 27 & 42\\
DeltaNet & 61 & 45\\
Lattice & \textbf{82} & \textbf{84}\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item At $L=2048$: +42 pp over TTT; +39 pp over DeltaNet.
\item At $d=128$, all three reach 100\%; small-width failures still exist.
\end{itemize}
\papersource{Paper, Fig. 7 and Appendix C.1, PDF pp. 24--25. Accuracy (\%); plotted fractions converted to percent. Some baselines are borrowed from the xLSTM paper.}
\end{frame}

\begin{frame}{Training scales better than attention at long lengths}
\centering
\includegraphics[width=.9\linewidth,height=.59\textheight,keepaspectratio]{figures/fig08-throughput.pdf}
\begin{itemize}
\item 16k, $C=64$: 158.582k tokens/s, \textbf{2.43$\times$} Transformer++.
\item Still \textbf{8.12\% slower} than TTT there; attention wins at 2k.
\end{itemize}
\papersource{Paper, Fig. 8, PDF p. 26; exact crop. 110M, $2\times2$ TPU Trillium, training throughput. TTT/Lattice use pure JAX; Transformer++ uses FlashAttention-2.}
\end{frame}

\begin{frame}{The best quality and speed are different settings}
\begin{columns}[T,onlytextwidth]
\column{.48\textwidth}
\begin{block}{Quality evidence}
Books long-context results: $C=1$.\\[2mm]
Most zero-shot results: $C=4$.\\[2mm]
Perplexity worsens as $C$ grows in the normalization ablation.
\end{block}
\column{.48\textwidth}
\begin{block}{Speed evidence}
Throughput curves: $C=16/64$.\\[2mm]
16k speedup vs. attention:\\
$1.07\times$ at $C=16$;\\
$2.43\times$ at $C=64$.
\end{block}
\end{columns}
\vspace{4mm}
\alert{A matched quality--throughput frontier is still needed.}
\begin{itemize}
\item No measured decoding latency or peak-memory comparison here.
\item Nonlinearity prevents standard inter-chunk associative scans.
\end{itemize}
\papersource{Paper, Appendix C, Table 5, Fig. 8, and limitations, PDF pp. 23--26, 28. Interpretation of the reported configurations.}
\end{frame}

\begin{frame}{Reproduction needs mathematical clarifications}
\begin{itemize}
\item \textbf{Gradient convention:} squared losses omit a factor 2 in some main-text updates; the appendix retains it.
\item \textbf{Projection norm:} Eq. (23) adds squared components, but projection geometry requires a subtraction; Listing 1 subtracts.
\item \textbf{Chunk normalization:} Listing 1 line 93 uses $\eta^2$ where norm preservation suggests $\mu^2$ for the old-state term.
\item \textbf{Implementation completeness:} \texttt{GLA\_intra\_chunk} is not defined in the listing; initialization is not fully specified.
\end{itemize}
\vspace{3mm}
\alert{These are reading checks, not evidence that the reported training results are invalid.}
\papersource{Paper, Eqs. (5)--(6), (23), (27), Listing 1, PDF pp. 5, 19, 21, 27--28. Independent derivation; downloaded author code was not executed.}
\end{frame}

\begin{frame}{The contribution is a better rule for limited-state writes}
\begin{enumerate}
\item Slot normalization yields an interpretable, state-dependent orthogonal update.
\item Fixed-state memory is used more effectively in several reported LM and recall settings.
\item Chunk approximation makes computation practical, with a quality--parallelism trade-off.
\end{enumerate}
\vspace{4mm}
\begin{block}{Discussion question}
At matched quality, parameters and memory, what chunk size makes Lattice preferable to a linear recurrence?
\end{block}
\papersource{Primary source: Mahdi Karami, Razvan Pascanu, Vahab Mirrokni. \emph{Lattice: Learning to Efficiently Compress the Memory}. \href{https://arxiv.org/abs/2504.05646v2}{arXiv:2504.05646v2}, 2025. PDF and matching author TeX, including Appendices A--D, were read for this presentation.}
\end{frame}

\appendix
\begin{frame}{Backup: DEC, ENC and TTT normalize different objects}
\renewcommand{\arraystretch}{1.45}
\begin{tabular}{lll}
\toprule
Model & Predicted quantity & Orthogonal to\\
\midrule
Lattice-DEC & $\phi(S)k$ & Each slot $s_i$\\
Lattice-ENC & $\phi(S)^\top v$ & Each slot $s_i$\\
TTT (paper view) & $\phi(Sk)$ & Decoder output $Sk$\\
\bottomrule
\end{tabular}
\vspace{3mm}
\[
\text{ENC:}\quad
\nabla_{s_i}\frac12\norm{\Phi^\top v-k}^2
=\frac{\phi_i^\top v-k_i}{\norm{s_i}}P(s_i)v.
\]
\begin{itemize}
\item Removing normalization from DEC recovers the delta rule.
\item The TTT comparison uses the paper's L2-normalization abstraction.
\end{itemize}
\papersource{Paper, Table 1; Eqs. (12)--(13), (18)--(19), (24)--(27), PDF pp. 7, 18--21.}
\end{frame}

\begin{frame}{Backup: the two chunk approximations are distinct}
Let $\tilde h_t=(S_0^b)^\top h_t$ and $G_t=\mu_t\one\beta_t^\top$.\par\medskip
\textbf{Eq. (15): freeze the state inside the write correction}
\[
S_t=G_t\odot S_{t-1}
+[\one(\tilde h_t\odot\hat k_t)^\top]\odot S_0^b
-h_t\hat k_t^\top.
\]
\textbf{Eq. (17): absorb the correction into a gate}
\[
S_t=[\one(\mu_t\beta_t+\tilde h_t\odot\hat k_t)^\top]\odot S_{t-1}
-h_t\hat k_t^\top.
\]
\begin{itemize}
\item Causal products $\Omega_{j,i,:}=a_j/a_i\,\mathbb I_{i\le j}$ unroll Eq. (15).
\item For $C>1$, instantaneous orthogonality and norm preservation need separate checking.
\end{itemize}
\papersource{Paper, Eqs. (15)--(17), (20)--(23), PDF pp. 8, 18--19. Use tokens $bC+1,\ldots,(b+1)C$ and state $S_{bC}$ to avoid the printed indexing ambiguity.}
\end{frame}

\begin{frame}{Backup: the projection norm has a negative cross term}
For $\norm{s}=1$, let $a=s^\top h$:
\[
\norm{h-as}^2=\norm{h}^2-2a(s^\top h)+a^2\norm{s}^2
=\norm{h}^2-a^2.
\]
\begin{block}{Independent check, not an author-reported experiment}
$s=(0.6,0.8)$, $h=(2,-1)$ gives $a=0.4$.\\[1mm]
Correct projected norm squared: $5-0.16=4.84$.\\
The positive-sum expression gives $5+0.16=5.16$.
\end{block}
With explicit decay and an orthogonal increment:
\[
\frac{\mu s+\Delta s}{\sqrt{\mu^2+\norm{\Delta s}^2}}
\quad\text{has unit norm.}
\]
\papersource{Compare Paper Eq. (23), PDF p. 19, with Listing 1 lines 89--95, p. 28. Listing subtraction is consistent with geometry; its old-state denominator is a separate question.}
\end{frame}

\begin{frame}{Backup: MQAR gains depend on width and difficulty}
\centering
\includegraphics[width=.9\linewidth,height=.74\textheight,keepaspectratio]{figures/fig07-mqar.pdf}
\papersource{Paper, Fig. 7, PDF p. 25; exact crop, all panels and legend. Rows: $L=1024/2048$. Columns: 48/96/256 KV pairs. Axis accuracy is a fraction despite the percent sign in the caption.}
\end{frame}

\begin{frame}{Backup: training details and coverage}
\centering
\renewcommand{\arraystretch}{1.35}
\begin{tabular}{lrrrr}
\toprule
Params / tokens & Blocks & Width & Head dim & Peak LR\\
\midrule
110M / 5B & 12 & 768 & 64 & $10^{-2}$\\
340M / 15B & 24 & 1024 & 64 & $1.5\cdot10^{-3}$\\
760M / 30B & 24 & 1536 & 64 & $1.25\cdot10^{-3}$\\
\bottomrule
\end{tabular}
\vspace{3mm}
\begin{itemize}
\item LM: AdamW, 0.5M tokens/batch, 1k-step warm-up, cosine decay; final LR $3\cdot10^{-5}$, weight decay 0.1, gradient clip 1.0.
\item MQAR: 100k training / 3k validation samples; 64 epochs; vocabulary 8192; separate learning-rate sweeps.
\end{itemize}
\papersource{Paper, Table 4 and Appendix C, PDF pp. 22--25. Tokenizer, full evaluation commands and multi-seed uncertainty are not sufficiently reported for an independent reproduction.}
\end{frame}
\end{document}