\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{Caution}{RGB}{161,76,34}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\readeranalysis}{\textcolor{Caution}{Reader's analysis}}
\newcommand{\R}{\mathbb R}
\title[LRKV]{Low-Rank Key Value Attention}
\subtitle{Shared projections, specialized heads, and the cost of a smaller cache}
\author{Paper authors: James O'Neill, Robert Clancy,\\Mariia Matskevichus, Fergal Reid\\[1ex]\small Independent paper discussion}
\date{arXiv:2601.11471v3 \; (7 April 2026)\\Reading date: 15 October 2026}

\begin{document}

\begin{frame}
\titlepage
\vspace{-1em}
\begin{center}
\small A shared K/V base plus per-head low-rank corrections can reduce cache storage.\\
The deployment claims require closer scrutiny than the linear algebra.
\end{center}
\end{frame}

\begin{frame}[t]{Can heads share storage while keeping distinct K/V features?}
\begin{block}{The design question}
How much of each head's key/value representation must be stored independently?
\end{block}
\vspace{.5em}
\begin{tabular}{lll}
\toprule
Mechanism & K/V structure & Cache elements / token / layer\\
\midrule
MHA & Independent per head & $2Hd_h$\\
MQA & One shared pair & $2d_h$\\
GQA & $n_{kv}$ shared groups & $2n_{kv}d_h$\\
LRKV & Shared base + residuals & $2(d_h+Hr)$\\
\bottomrule
\end{tabular}
\vspace{.6em}
\begin{itemize}
\item $H$: query heads; $d_h$: head width; $r$: residual rank.
\item Queries remain independent in all four designs.
\end{itemize}
\papersource{Paper, Secs. 1--3, PDF pp. 1--4. Cache excludes model parameters.}
\end{frame}

\begin{frame}[t]{LRKV constrains differences between heads}
\[
\boxed{W_h^{K,V}=W_{\mathrm{shared}}^{K,V}+U_h^{K,V}(B_h^{K,V})^\top}
\]
\begin{columns}[T,onlytextwidth]
\begin{column}{.47\textwidth}
\begin{block}{Shared component}
$W_{\mathrm{shared}}\in\R^{d\times d_h}$\\[.4em]
Dense, with no rank constraint.\\[.4em]
One common projection across heads.
\end{block}
\end{column}
\begin{column}{.49\textwidth}
\begin{block}{Head-specific correction}
$U_h\in\R^{d\times r}$, $B_h\in\R^{d_h\times r}$\\[.4em]
$\operatorname{rank}(U_hB_h^\top)\le r$.\\[.4em]
Separate factors for keys and values.
\end{block}
\end{column}
\end{columns}
\vspace{.5em}
The effective $W_h$ may be full rank, while the family of heads is constrained.
\papersource{Paper, Sec. 3, Eqs. (2)--(3), PDF p. 3. Here $d=Hd_h$.}
\end{frame}

\begin{frame}[t]{Training learns the base and residuals together}
\begin{itemize}
\item Jointly train $W_{\mathrm{shared}}$, $U_h$, and $B_h$ from scratch.
\item Initialize residuals near the shared baseline: about $0.1\|W_{\mathrm{shared}}\|_F$.
\item Keep full, independent query projections $W_h^Q$.
\item Materialize full per-head K/V during training.
\end{itemize}
\vspace{.5em}
\begin{block}{Two different notions of equivalence}
Factored and expanded evaluation of the \emph{same LRKV model} are equivalent in the linear setting. A low-rank LRKV model does not represent every possible MHA parameterization.
\end{block}
\papersource{Paper, Sec. 3, training and initialization, PDF p. 3. Cache savings target decoding.}
\end{frame}

\begin{frame}[t]{Cache shared features and compact per-head latents}
\begin{columns}[T,onlytextwidth]
\begin{column}{.50\textwidth}
\begin{block}{Once per layer}
\[
K_s=XW_{\mathrm{shared}}^K,\qquad
V_s=XW_{\mathrm{shared}}^V
\]
Each tensor: $L\times d_h$.
\end{block}
\end{column}
\begin{column}{.46\textwidth}
\begin{block}{For each head}
\[
R_h^K=XU_h^K,\quad R_h^V=XU_h^V
\]
Each tensor: $L\times r$.
\end{block}
\end{column}
\end{columns}
\vspace{.5em}
The implied full features are
\[
K_h=K_s+R_h^K(B_h^K)^\top,\qquad
V_h=V_s+R_h^V(B_h^V)^\top.
\]
\begin{itemize}
\item $L$: cached sequence length; $X\in\R^{L\times d}$.
\item $B_h^{K,V}$ are weights, not per-token cache entries.
\end{itemize}
\papersource{Paper, Sec. 3, Eqs. (4)--(5), PDF p. 3. $K_s,V_s$ abbreviate the shared features.}
\end{frame}

\begin{frame}[t]{Compute logits without expanding historical keys}
For the current query $q_h\in\R^{1\times d_h}$:
\[
\underbrace{q_hK_h^\top}_{1\times L}
=\underbrace{q_hK_s^\top}_{\text{shared match}}
+\underbrace{(q_hB_h^K)(R_h^K)^\top}_{\text{head-specific correction}}.
\]
\vspace{.6em}
\begin{enumerate}
\item Project the \emph{current query} into $r$ dimensions.
\item Compare it with the cached residual latents.
\item Add both score paths, then apply scaling and softmax.
\end{enumerate}
\[
a_h=\operatorname{softmax}\!\left(\frac{q_hK_h^\top}{\sqrt{d_h}}+\text{causal mask}\right).
\]
\papersource{Paper, Eq. (6), PDF p. 4. Exact for the stated linear projection model.}
\end{frame}

\begin{frame}[t]{Aggregate values before the residual up-projection}
For attention weights $a_h\in\R^{1\times L}$:
\[
\boxed{o_h=a_hV_s+(a_hR_h^V)(B_h^V)^\top}
\]
\begin{columns}[T,onlytextwidth]
\begin{column}{.47\textwidth}
\begin{block}{Shared path}
$a_hV_s$ produces one $d_h$-dimensional output.
\end{block}
\end{column}
\begin{column}{.49\textwidth}
\begin{block}{Residual path}
First reduce $L$ latents to one $r$-vector; then map that vector to $d_h$.
\end{block}
\end{column}
\end{columns}
\vspace{.6em}
\begin{itemize}
\item Historical per-head K/V need not be materialized.
\item Dense softmax and the scan over past tokens remain.
\end{itemize}
\papersource{Paper, Eq. (7), PDF p. 4. Position encoding and normalization require additional treatment.}
\end{frame}

\begin{frame}[t]{The rank directly sets the theoretical cache budget}
\[
\rho=\frac{2L(d_h+Hr)}{2LHd_h}=\boxed{\frac1H+\frac r{d_h}}.
\]
\centering
\begin{tabular}{lrrr}
\toprule
Scale label & $H$ & $r$ (Table 7) & Recomputed cache / MHA\\
\midrule
128M & 6 & 46 & 52.60\%\\
1.2B & 12 & 51 & 48.18\%\\
2.5B & 18 & 55 & 48.52\%\\
6.3B & 32 & 54 & 45.31\%\\
\bottomrule
\end{tabular}
\par\raggedright\vspace{.6em}
All rows use $d_h=128$. At 2.5B: 4,608 $\to$ 2,236 elements per token per layer.
\begin{itemize}
\item $r=0$: MQA-style sharing.
\item $r=d_h$: expressive enough for MHA, but cache ratio is $1+1/H$.
\end{itemize}
\papersource{Paper, Eq. (9), p. 4; Table 7, p. 24. Percentages recomputed from those configurations.}
\end{frame}

\begin{frame}[t]{Smaller storage trades bandwidth for extra arithmetic}
\begin{block}{Per head, per decoding step}
MHA's main scan: $O(Ld_h)$\hfill
LRKV's additional work: $O(Lr+rd_h)$.
\end{block}
\vspace{.4em}
\begin{itemize}
\item The residual scan costs roughly $r/d_h$ of the main attention scan at long context.
\item Shared cache reuse may reduce memory traffic; actual latency depends on the kernel.
\item Cache savings and K/V parameter savings have different formulas:
\end{itemize}
\[
P_{\mathrm{LRKV,KV}}=2dd_h+2Hr(d+d_h),\quad
P_{\mathrm{MHA,KV}}=2Hdd_h.
\]
\papersource{Paper, Sec. 3 and Eq. (10), PDF p. 4. No end-to-end decoding latency table is provided.}
\end{frame}

\begin{frame}[t]{The experiments change attention within a shared training recipe}
\centering
\begin{tabular}{lrrl}
\toprule
Scale label & Context & Tokens & Role\\
\midrule
128M & 2,048 & 100B & Main comparison + rank sweep\\
1.2B / 2.5B / 6.3B & 2,048 & 50B each & Scaling comparison\\
512M & 8,192 & 50B & Long-context training\\
\bottomrule
\end{tabular}
\par\raggedright\vspace{.7em}
\begin{itemize}
\item FineWeb-Edu; 8 H200 GPUs; bfloat16; Muon + AdamW.
\item Baselines: MHA, GQA, MQA, and MLA, with different cache budgets.
\item Scale labels refer to the MHA base, not equal total parameters.
\item Downstream: one epoch of supervised midtraining on 568K examples.
\end{itemize}
\papersource{Paper, Sec. 4; Appendix D, Tables 5--7, PDF pp. 23--24. Architecture inconsistencies discussed later.}
\end{frame}

\begin{frame}[t]{Table 1 reports the lowest final CE at every scale}
\centering
\begin{tabular}{lrrrr}
\toprule
Method & 128M & 1.2B & 2.5B & 6.3B\\
\midrule
MHA & 2.903 & 2.530 & 2.389 & 2.319\\
GQA & 2.918 & 2.536 & 2.397 & 2.354\\
MQA & 2.929 & 2.573 & 2.408 & 2.304\\
MLA & 2.901 & 2.564 & 2.392 & 2.377\\
\textbf{LRKV} & \textbf{2.893} & \textbf{2.509} & \textbf{2.376} & \textbf{2.288}\\
\bottomrule
\end{tabular}
\par\raggedright\vspace{.8em}
\begin{itemize}
\item Held-out cross-entropy; lower is better. Compare within columns.
\item Relative CE reduction vs. MHA: 0.34\%, 0.83\%, 0.54\%, 1.34\%.
\item At 6.3B, MQA also beats MHA: stronger sharing need not worsen CE.
\end{itemize}
\papersource{Paper, Table 1, PDF p. 5. Relative changes are calculated from the reported CE values.}
\end{frame}

\begin{frame}[t]{LRKV reaches baseline quality in fewer training steps}
\centering
\includegraphics[width=.91\linewidth,height=.54\textheight,keepaspectratio]{figures/fig03-efficiency.pdf}
\par\raggedright\vspace{.3em}
At 2.5B: 18.7\% fewer steps to MHA's final loss; 23.6\% average across four baselines.
\medskip
The paper's footnote identifies \alert{sample efficiency}, not a measured per-step or wall-clock speedup.
\papersource{Original Fig. 3, PDF p. 6; compute-accounting footnote, p. 5. Cache percentages are as reported.}
\end{frame}

\begin{frame}[t]{At 2.5B, reasoning gains coexist with a code-generation drop}
\centering
\begin{tabular}{lrrr}
\toprule
Task & MHA (\%) & LRKV (\%) & Difference (pp)\\
\midrule
ARC-Easy & 66.6 & 70.7 & +4.1\\
ARC-Challenge & 47.1 & 53.8 & +6.7\\
MMLU & 39.3 & 42.2 & +2.9\\
GSM8K & 10.2 & 11.3 & +1.1\\
HumanEval & 13.4 & 11.7 & \textcolor{Caution}{$-1.7$}\\
\bottomrule
\end{tabular}
\par\raggedright\vspace{.7em}
\begin{itemize}
\item After the same supervised midtraining mixture.
\item Reported five-task average: 35.3\% $\to$ 37.9\% (+2.6 pp).
\item The gain is an average advantage, with task-specific trade-offs.
\end{itemize}
\papersource{Paper, Table 2, PDF p. 6; midtraining mixture, Appendix D.2, p. 24.}
\end{frame}

\begin{frame}[t]{Average downstream gains persist, but exceptions matter}
\centering
\begin{tabular}{lrrr}
\toprule
Reported five-task average (\%) & 128M & 2.5B & 6.3B\\
\midrule
MHA & 17.9 & 35.3 & 38.8\\
MQA & 17.3 & 33.3 & 33.4\\
MLA & 18.0 & 36.9 & 36.9\\
\textbf{LRKV} & \textbf{18.9} & \textbf{37.9} & \textbf{40.2}\rlap{$^{*}$}\\
\bottomrule
\end{tabular}
\par\raggedright\vspace{.6em}
\begin{itemize}
\item 6.3B HumanEval: LRKV 12.8\%, MHA 14.6\%.
\item 128M GSM8K: LRKV 0.3\%, MHA 1.0\%.
\item Five 2.5B architectures show BPB/score association ($R^2=0.786$); this does not identify a causal mechanism.
\end{itemize}
\papersource{Table 2, p. 6; Fig. 19, p. 26. $^{*}$Reported 40.2\%; the five listed scores average to 40.36\%.}
\end{frame}

\begin{frame}[t]{At 8K context, LRKV reports a 2.74\% CE reduction}
\begin{columns}[T,onlytextwidth]
\begin{column}{.60\textwidth}
\centering
\includegraphics[width=\linewidth,height=.66\textheight,keepaspectratio]{figures/fig04-long-context.pdf}
\end{column}
\begin{column}{.37\textwidth}
\begin{tabular}{lr}
\toprule
Method & CE $\downarrow$\\
\midrule
MHA & 2.740\\
GQA & 2.704\\
MQA & 2.728\\
MLA & 2.714\\
LRKV & \textbf{2.665}\\
\bottomrule
\end{tabular}
\vspace{.8em}

512M label; 50B tokens.\\[.5em]
$r=51$; 48.2\% reported cache.\\[.5em]
Fixed-context language modeling.
\end{column}
\end{columns}
\papersource{Original Fig. 4, p. 6; precise values from Table 9, p. 26. This does not test context extrapolation.}
\end{frame}

\begin{frame}[t]{Table 8 shows a trade-off, not universal dominance over MLA}
\centering
\includegraphics[width=.91\linewidth,height=.64\textheight,keepaspectratio]{figures/table08-frontier.pdf}
\par\raggedright\vspace{.3em}
MLA-384: 25.0\% cache, CE 2.894; LRKV-32: 41.7\%, CE 2.896.
\papersource{Redrawn from Table 8, PDF p. 25 (128M, 100B tokens). Original Fig. 5 contains conflicting MLA points.}
\end{frame}

\begin{frame}[t]{Head diversity is measured in bilinear-form space}
\[
A_h=W_h^Q(W_h^K)^\top,\qquad
x_iA_hx_j^\top=\text{unscaled linear attention logit}.
\]
\begin{itemize}
\item Coupled orthogonal changes of Q/K coordinates leave $A_h$ unchanged.
\item Form the Gram matrix of normalized $A_h$; center across heads.
\end{itemize}
\[
G_c=JGJ,\qquad J=I-\frac{\mathbf1\mathbf1^\top}{H},
\]
\[
p_i=\frac{\lambda_i(G_c)}{\sum_j\lambda_j(G_c)},\qquad
r_{\mathrm{eff}}=\exp\!\left(-\sum_i p_i\ln p_i\right).
\]
\papersource{Paper, Sec. 4.3, pp. 7--9; Appendix C.1, p. 15. Static parameter metric; excludes V and input behavior.}
\end{frame}

\begin{frame}[t]{LRKV approaches the centered-rank ceiling}
\centering
\begin{tabular}{lrr}
\toprule
PCA effective rank / $H$ & 128M ($H=6$) & 2.5B ($H=18$)\\
\midrule
MHA & 82.4\% & 94.0\%\\
LRKV ($r=64$) & 83.0\% & 93.5\%\\
MQA & 78.4\% & 91.0\%\\
MLA & 78.5\% & 91.6\%\\
\midrule
\textit{Mathematical ceiling} & \textit{83.33\%} & \textit{94.44\%}\\
\bottomrule
\end{tabular}
\par\raggedright\vspace{.6em}
\readeranalysis: $G_c\mathbf1=0$, so $r_{\mathrm{eff}}\le H-1$.
\begin{itemize}
\item Different head counts change the percentage ceiling.
\item MQA retains diversity through independent queries; centering increases its 2.5B percentage from 86.2 to 91.0.
\end{itemize}
\papersource{Paper, Table 3, p. 16. Ceiling derived here. The $r=64$ analysis differs from Table 7's main ranks.}
\end{frame}

\begin{frame}[t]{Large weight norms motivate checking normalization placement}
\centering
\includegraphics[width=.95\linewidth,height=.52\textheight,keepaspectratio]{figures/fig08-magnitude.pdf}
\par\raggedright\vspace{.4em}
\begin{itemize}
\item Authors attribute larger LRKV norms to additive optimization and post-projection RMSNorm.
\item RMSNorm of a sum is not the sum of separately normalized paths.
\item Factored decoding needs consistent per-token, per-head scaling.
\end{itemize}
\papersource{Original Fig. 8 and normalization discussion, PDF p. 9. Scaling requirements are reader's analysis.}
\end{frame}

\begin{frame}[t]{Full RoPE requires more than linearity}
\readeranalysis\quad Use column vectors for the residual key $B_hz_t$.
\[
P_t(k_{s,t}+B_hz_t)=P_tk_{s,t}+P_tB_hz_t.
\]
To rotate only the low-dimensional latent, we would need
\[
\boxed{P_tB_h=B_h\widetilde P_t\quad\text{for each position }t.}
\]
\begin{itemize}
\item $P_t$ rotates $d_h$ dimensions; $\widetilde P_t$ would act in $r$ dimensions.
\item This requires an invariant residual subspace, absent for arbitrary learned $B_h$.
\item The paper's full-RoPE claim therefore does not follow from Eqs. (6)--(7).
\end{itemize}
\papersource{Claim examined: Sec. 3, positional-embedding paragraph, PDF p. 4. A counterexample is in the backup slides.}
\end{frame}

\begin{frame}[t]{The measured-cache appendix mixes incompatible numbers}
\centering
\begin{tabular}{lrrr}
\toprule
Scale & LRKV / MHA in Table 12 & Actual ratio & Printed ratio\\
\midrule
128M & 48 / 72 MB & 66.67\% & 52.6\%\\
2.5B & 360 / 648 MB & 55.56\% & 48.4\%\\
6.3B & 612 / 1152 MB & 53.13\% & 45.1\%\\
\bottomrule
\end{tabular}
\par\raggedright\vspace{.6em}
\begin{itemize}
\item Table 12 says $r=64$; the actual ratios match that rank.
\item Main configurations use $r=46,51,55,54$ instead.
\item ``Measured'' means tensor element counts $\times$ bytes per element.
\item Layer counts also conflict: 2.5B is 18 layers in Table 5, 36 in analysis figures.
\end{itemize}
\papersource{Paper, Appendix G, pp. 28--29; Table 5, p. 23; Figs. 7--8, pp. 8--9. Ratios calculated here.}
\end{frame}

\begin{frame}[t]{The architecture idea is clearer than the deployment evidence}
\begin{block}{What is useful}
A shared dense base plus low-rank head corrections gives an interpretable capacity--cache control. Associativity makes the linear formulation exact without historical K/V expansion.
\end{block}
\begin{block}{What the tables report}
Better pretraining CE and higher average downstream scores under the reported training recipe, with task-specific regressions.
\end{block}
\begin{block}{What a deployment claim still needs}
A consistent RoPE/normalization implementation, reconciled configurations and memory figures, and optimized comparisons at matched cache budgets.
\end{block}
\papersource{Synthesis of Secs. 3--4 and Appendices C, D, F, G. Interpretation by the reader.}
\end{frame}

\begin{frame}[t]{References and discussion prompts}
James O'Neill, Robert Clancy, Mariia Matskevichus, and Fergal Reid.\\[.4em]
\textit{Low-Rank Key Value Attention}. arXiv:2601.11471v3, 2026.\\[.4em]
\href{https://arxiv.org/abs/2601.11471v3}{https://arxiv.org/abs/2601.11471v3}
\vspace{1em}
\begin{enumerate}
\item How should residual subspaces be constrained to support full RoPE?
\item At equal cache size, when is a shared dense base preferable to a joint latent?
\item Do operator-space diversity measures predict task-specific behavior?
\end{enumerate}
\papersource{Original figures reproduced from the v3 source, CC BY 4.0. Table 8 plot redrawn with attribution.}
\end{frame}

\appendix

\begin{frame}[t]{Backup: the complete rank sweep clarifies the trade-off}
\centering
\begin{tabular}{lrrr}
\toprule
Configuration & Cache / MHA & CE $\downarrow$ & BPB $\downarrow$\\
\midrule
LRKV $r=8$ & 22.9\% & 2.908 & 0.881\\
LRKV $r=16$ & 29.2\% & 2.905 & 0.880\\
LRKV $r=32$ & 41.7\% & 2.896 & 0.877\\
LRKV $r=64$ & 66.7\% & 2.888 & 0.875\\
LRKV $r=128$ & 116.7\% & 2.877 & 0.873\\
MLA $d_c=128$ & 8.3\% & 2.901 & 0.876\\
MLA $d_c=192$ & 12.5\% & 2.898 & 0.876\\
MLA $d_c=384$ & 25.0\% & 2.894 & 0.875\\
MHA & 100.0\% & 2.903 & 0.878\\
\bottomrule
\end{tabular}
\papersource{Paper, Table 8, PDF p. 25. 128M label, 100B tokens, $H=6$, $d_h=128$. The printed 12-5\% is read as 12.5\%.}
\end{frame}

\begin{frame}[t]{Backup: a rotation can leave the residual subspace}
\readeranalysis\quad Let the residual span only the first coordinate:
\[
B=\begin{bmatrix}1\\0\end{bmatrix},\qquad
P=\begin{bmatrix}0&-1\\1&0\end{bmatrix},\qquad
PB=\begin{bmatrix}0\\1\end{bmatrix}.
\]
For any scalar $c$ acting on the one-dimensional latent,
\[
Bc=\begin{bmatrix}c\\0\end{bmatrix}\ne PB.
\]
\begin{itemize}
\item Linearity distributes rotation over addition, not through an arbitrary $B$.
\item Pair-aligned or otherwise invariant subspaces could change the result.
\item Such constraints would need their own capacity and cache analysis.
\end{itemize}
\papersource{Teaching counterexample to the unconditional claim on PDF p. 4; not a model experiment.}
\end{frame}

\begin{frame}[t]{Backup: centering itself removes one available dimension}
For $H$ perfectly orthogonal normalized operators, $G=I_H$:
\[
G_c=JGJ=J,\quad \lambda(G_c)=\underbrace{1,\ldots,1}_{H-1\text{ times}},0.
\]
\[
r_{\mathrm{eff}}(G)=H,\qquad r_{\mathrm{eff}}(G_c)=H-1.
\]
\begin{itemize}
\item With 18 heads, the percentage falls from 100\% to 94.44\% even in this ideal case.
\item Therefore a roughly five-point decrease alone does not identify a shared mean component.
\item Figure 9 is labeled PCA but reaches nearly 6 for six heads; a centered rank cannot exceed 5.
\end{itemize}
\papersource{Reader's derivation from Appendix C.1, p. 15. Figure 9 and Table 3 are on p. 16.}
\end{frame}

\begin{frame}[t]{Backup: MLA also admits associative computation}
With $Z=XW_{\mathrm{down}}$ and $K=ZW_{\mathrm{up}}^K$:
\[
qK^\top=\bigl(q(W_{\mathrm{up}}^K)^\top\bigr)Z^\top,
\qquad
aV=(aZ)W_{\mathrm{up}}^V.
\]
\begin{itemize}
\item This feature bottleneck retains every token position.
\item Full historical K/V reconstruction is not necessary in this linear form.
\item Appendix F describes association but Table 10 charges MLA for reconstruction.
\item Compare optimized kernels under consistent positional-encoding assumptions.
\end{itemize}
\papersource{Paper, Appendix F, PDF pp. 27--28. Equations rewritten with dimensionally explicit transposes.}
\end{frame}

\begin{frame}[t]{Backup: training and evaluation details affect interpretation}
\begin{columns}[T,onlytextwidth]
\begin{column}{.47\textwidth}
\begin{block}{Pretraining}
FineWeb-Edu; 32,768 tokens/batch.\\[.5em]
Muon + AdamW; cosine decay; 2,000 warmup steps.\\[.5em]
Weight decay 0.1; clipping 1.0.
\end{block}
\end{column}
\begin{column}{.49\textwidth}
\begin{block}{Supervised midtraining}
460K SmolTalk + 100K MMLU auxiliary + 8K GSM8K.\\[.5em]
One epoch; 524,288 tokens/batch; context 2,048.
\end{block}
\end{column}
\end{columns}
\vspace{.6em}
Per-task shot counts and code-sampling details are not specified individually. BPB also requires a byte-count denominator; CE/$\ln2$ alone gives bits per token.
\papersource{Paper, Appendix D, pp. 23--26; Appendix F.1, p. 27. Metric interpretation by the reader.}
\end{frame}

\end{document}