% Self-authored research reading deck; based on the arxiv-paper-reader template.
% Compile using: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{Muted}{RGB}{85,97,108}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\setbeamerfont{block title}{size=\normalsize}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{Muted}#1\par}}
\newcommand{\R}{\mathbb R}
\newcommand{\rowflat}{\operatorname{flat}}
\newcommand{\prem}{W_g^{\mathrm{premix}}}
\newcommand{\pp}{\text{ pp}}
\title[VHA]{Virtual Head Attention}
\subtitle{A structured training parameterization\\that deploys as standard GQA-2}
\author{Paper by Xibo Ding, Guoxia Wang, Jinle Zeng, Jiabin Yang,\\
Sirui Tao, Chao Yang, and Dianhai Yu\\[2mm]
\small Baidu Inc.\ and Peking University\\[3mm]
\small Independent research reading presentation}
\date{arXiv:2610.17959v1 \quad|\quad 19 October 2026\\
\small Main discussion: approximately 25 minutes; technical backups follow}

\begin{document}
% 1
\begin{frame}
\titlepage
\end{frame}

% 2
\begin{frame}{The target is quality at a small KV budget}
\begin{block}{Research question}
Can two KV heads retain GQA quality, reduce training arithmetic, and use standard serving kernels?
\end{block}
\medskip
\centering
\begin{tabular}{@{}lrrr@{}}
\toprule
1.7B model & KV KB/layer/token & Train G/token & Avg.\ accuracy \\
\midrule
GQA-8 & 4.00 & 10.61 & 54.09 \\
MLA & 1.12 & 10.42 & 52.49 \\
MFA & 1.00 & 15.90 & 54.06 \\
\textbf{VHA} & \textbf{1.00} & \textbf{9.78} & \textbf{54.50} \\
\bottomrule
\end{tabular}\par
\medskip
\raggedright
50B training tokens; 4K context; eight-task zero-shot average (\%).
\papersource{Paper, Tables 1 and 5, PDF pp. 6 and 8. KV is BF16; KB uses 1024 bytes.}
\end{frame}

% 3
\begin{frame}{Two linear modules surround ordinary attention}
\centering
\includegraphics[width=0.92\linewidth,height=0.76\textheight,keepaspectratio]{figures/fig-01-architecture.pdf}
\papersource{Paper, Fig. 1(a,b), PDF p. 4; cropped from the original vector figure.}
\end{frame}

% 4
\begin{frame}{Eight physical heads become sixteen virtual heads}
\[
H_q'=H_q/2,\qquad H_{kv}=2,\qquad H_v=H_{kv}H_q'=H_q.
\]
\centering
\begin{tabular}{@{}lrrr@{}}
\toprule
1.7B configuration & Physical Q & Attention Q & KV heads \\
\midrule
GQA baseline & 16 & 16 & 8 \\
VHA during training & 8 & 16 & 2 \\
VHA after absorption & 16 & 16 & 2 \\
\bottomrule
\end{tabular}\par
\medskip
\raggedright
\begin{itemize}
\item Each of two KV groups receives a transformed copy of all eight physical query heads.
\item The attention operation still computes sixteen query heads.
\item The paper's VHA 8-2-2 labels the training projections.
\end{itemize}
\papersource{Paper, Sec. 3.2, PDF p. 4; Algorithm 1, PDF p. 20.}
\end{frame}

% 5
\begin{frame}{Premix specializes a shared query basis by KV group}
\[
Q_i=XW_Q^{(i)},\qquad
\widetilde Q_{i,g}=Q_i\prem,
\qquad \prem\in\R^{d\times d}.
\]
\begin{itemize}
\item Transform the \alert{feature dimension} within each physical head.
\item Share one transform across all physical heads in the same KV group.
\item Initialize with an identity anchor: $\prem=I+\alpha\epsilon$.
\end{itemize}
\begin{block}{1.7B dimensions}
$X:S\times2048$;\quad $W_Q^{(i)}:2048\times128$;\\
two $128\times128$ group transforms; eight physical heads.
\end{block}
\papersource{Paper, Sec. 3.3, Eqs. (1)--(3), PDF pp. 4--5; Table 12, p. 18.}
\end{frame}

% 6
\begin{frame}{Postmix adds a low-rank residual on the head axis}
For one token, $Y\in\R^{H_v\times d}$ contains all attention-head outputs.
\[
Y'=Y+B(A^\top Y)=(I+BA^\top)Y,\qquad
A,B\in\R^{H_v\times r}.
\]
\begin{itemize}
\item $A^\top$: compress head information into $r$ channels.
\item $B$: distribute it back across heads, including other KV groups.
\item $B=0$ at initialization, so the initial map is the identity.
\end{itemize}
\begin{block}{The low-rank object is the residual}
$\operatorname{rank}(BA^\top)\le r$; the full $I+BA^\top$ can be full rank.\\
At 1.7B: $H_v=16$, $r=4$.
\end{block}
\papersource{Paper, Sec. 3.4, Eq. (4), PDF p. 5.}
\end{frame}

% 7
\begin{frame}{Both modules merge exactly into standard projections}
\[
\widetilde W_Q^{(i,g)}=W_Q^{(i)}\prem,
\qquad W_Q^{\mathrm{abs}}=\operatorname{concat}_{g,i}
\bigl(\widetilde W_Q^{(i,g)}\bigr).
\]
\[
W_O^{\mathrm{abs}}=
\bigl((I+AB^\top)\otimes I_d\bigr)W_O.
\]
\begin{itemize}
\item Premix is before RoPE; retain that order after merging.
\item $BA^\top$ becomes $AB^\top$ because flattening transposes the head mixing.
\item Discard the factor matrices and run ordinary GQA with two KV heads.
\end{itemize}
\begin{block}{Meaning of zero extra deployment overhead}
No additional Premix/Postmix operators relative to the same-shape GQA-2.
\end{block}
\papersource{Paper, Eq. (5), p. 5; Algorithm 1 and Eqs. (14)--(18), PDF p. 20.}
\end{frame}

% 8
\begin{frame}{Forward equivalence does not imply identical training}
\begin{columns}[T]
\column{0.48\textwidth}
\begin{block}{Premix}
Shared physical heads and group transforms couple gradients.
\[
\widetilde W_Q^{(i,g)}=W_Q^{(i)}\prem
\]
The query family is structurally constrained.
\end{block}
\column{0.48\textwidth}
\begin{block}{Postmix}
The ordinary output projection already combines all heads.
\[
W_O^{\mathrm{abs}}=
((I+AB^\top)\otimes I_d)W_O
\]
Extra factors change the optimization path.
\end{block}
\end{columns}
\medskip
\alert{Interpretation:} beneficial parameter sharing and optimization bias, rather than a larger deployed attention function class.
\papersource{Paper, Sec. 2.3, p. 3; Appendices G--H, PDF pp. 17--21.}
\end{frame}

% 9
\begin{frame}{Factorized compute savings change after absorption}
\centering
\begin{tabular}{@{}lrr@{}}
\toprule
1.7B Q path, per layer & Parameters & Forward FLOPs/token \\
\midrule
Ordinary 16-head Q & 4.194M & 8.389M \\
VHA physical Q + Premix & 2.130M & 4.719M \\
Absorbed VHA Q & 4.194M & 8.389M \\
\bottomrule
\end{tabular}\par
\medskip
\raggedright
\begin{itemize}
\item Q parameters fall by 49.2\%; factorized Q FLOPs fall by 43.75\%.
\item $H_v$ is unchanged: the quadratic attention term remains.
\item Table 5's 7.9\% reduction concerns total training FLOPs.
\item Fully merged GQA prefill expands Q again; the KV savings remain.
\end{itemize}
\papersource{Reader calculation from Sec. 3.5 and Appendix G dimensions, pp. 5--6, 18.\\
Reported total training cost: Table 5, PDF p. 8.}
\end{frame}

% 10
\begin{frame}{Three evaluation tracks test different claims}
\begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.23\linewidth}>{\raggedright\arraybackslash}p{0.33\linewidth}>{\raggedright\arraybackslash}p{0.36\linewidth}@{}}
\toprule
Track & Training setup & What it tests \\
\midrule
0.6B / 1.7B / 4B / 8B-A1B &
50B FineWeb-Edu tokens; 4K; BF16 &
Eight-task quality and arithmetic across scale \\
\addlinespace[8pt]
30B-A3B MoE &
500B internal-corpus tokens; 8K; FP8 &
Four-category macro-average over 12 benchmarks \\
\addlinespace[8pt]
10B-A1B MoE &
500B at 8K + 100B at 32K + 100B at 128K &
Long-context behavior in a matched hybrid backbone \\
\bottomrule
\end{tabular}\par
\medskip
The 30B and long-context results use separate model families and recipes.
\papersource{Paper, Sec. 4.1, PDF p. 6; Appendices A and D, pp. 14, 18.}
\end{frame}

% 11
\begin{frame}{VHA has the highest mean at all four tested scales}
\centering
\begin{tabular}{@{}lrrrrrr@{}}
\toprule
Scale & MHA & GQA & MLA & MFA & VHA & $\Delta$ vs.\ GQA \\
\midrule
0.6B & 50.22 & 50.42 & 49.52 & 50.66 & \textbf{51.04} & +0.62 \\
1.7B & 53.24 & 54.09 & 52.49 & 54.06 & \textbf{54.50} & +0.41 \\
4B & 56.68 & 56.90 & 56.38 & 56.94 & \textbf{57.71} & +0.81 \\
8B-A1B & 54.60 & 54.86 & 53.79 & 54.81 & \textbf{54.91} & +0.05 \\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Eight-task accuracy (\%); differences are percentage points.
\item VHA cache: 1 KB per layer/token at all four scales.
\item GQA cache: 4 KB for dense models, 2 KB for the MoE.
\item The 8B-A1B gain is small; no repeated-run uncertainty is given at that scale.
\end{itemize}
\papersource{Paper, Table 1, PDF p. 6. Parameter-matched configurations use different head counts by method.}
\end{frame}

% 12
\begin{frame}{The compressed baseline isolates the recovery}
\centering
\begin{tabular}{@{}lrrr@{}}
\toprule
1.7B configuration & Actual attention Q & Avg. & $\Delta$ vs.\ GQA-8 \\
\midrule
GQA 16-8-8 & 16 & 54.09 & --- \\
GQA 16-2-2 & 16 & 53.09 & $-1.00$ \\
GQA 8-2-2 & 8 & 52.38 & $-1.71$ \\
Premix only & 16 & 53.31 & $-0.78$ \\
Postmix only & \multicolumn{1}{c}{---} & 53.95 & $-0.14$ \\
\textbf{Full VHA} & \textbf{16} & \textbf{54.50} & \textbf{+0.41} \\
\bottomrule
\end{tabular}\par
\medskip
\raggedright
\alert{Same deployed shape:} VHA beats GQA 16-2-2 by 1.41 points.\\
At 4B, Postmix-only already beats GQA by 0.32 points; the full model is best at +0.81.
\papersource{Paper, Tables 2 and 4, PDF pp. 7--8. Postmix-only attention-head expansion is not specified precisely.}
\end{frame}

% 13
\begin{frame}{The 30B result is a four-category macro-average}
\centering
\begin{tabular}{@{}lrrr@{}}
\toprule
Category / metric & GQA & MLA & VHA \\
\midrule
General & 52.86 & 55.57 & \textbf{56.25} \\
Math \& STEM & 43.09 & 43.49 & \textbf{46.40} \\
Coding & 51.85 & \textbf{54.37} & 53.23 \\
Multilingual & 43.26 & 45.20 & \textbf{45.56} \\
\midrule
Four-category mean & 47.76 & 49.66 & \textbf{50.36} \\
KV KB/layer/token & 2.00 & 1.12 & \textbf{1.00} \\
\bottomrule
\end{tabular}\par
\medskip
\raggedright
VHA: +2.60 points vs.\ GQA; +0.70 vs.\ MLA.\\
HumanEval+ is a negative case: 47.56 vs.\ 50.00 / 52.44.
\papersource{Paper, Table 3, PDF p. 7; Table 11, p. 17. 500B tokens; 8K; FP8 training; BF16 cache accounting.}
\end{frame}

% 14
\begin{frame}{Long-context validation includes 200B extra tokens}
\begin{block}{Matched 10B-A1B hybrid backbone}
Full attention : sliding-window attention = 1 : 3.\\
Only full-attention layers change: GQA-4 $\to$ VHA-2.\\
Sliding-window layers keep eight KV heads in both models.
\end{block}
\centering
\begin{tabular}{@{}lrrr@{}}
\toprule
Stage & Tokens in stage & Cumulative steps & RoPE base \\
\midrule
8K pretraining & 500B & 60K & $10^4$ \\
32K mid-training & 100B & 72K & $6.4\times10^5$ \\
128K mid-training & 100B & 96K & $5\times10^6$ \\
\bottomrule
\end{tabular}\par
\medskip
\raggedright
KV halves in full-attention layers; the total hybrid cache ratio also depends on the unchanged sliding-window component.
\papersource{Paper, Appendix A.1, PDF p. 14.}
\end{frame}

% 15
\begin{frame}{RULER gains are largest in variable tracking}
\centering
\begin{tabular}{@{}lrrrr@{}}
\toprule
& \multicolumn{2}{c}{RULER Overall} & \multicolumn{2}{c}{Variable Tracking} \\
Length & GQA & VHA & GQA & VHA \\
\midrule
8K & 79.1 & \textbf{90.1} & 44.6 & 94.0 \\
16K & 78.2 & \textbf{88.5} & 28.4 & 93.4 \\
32K & 72.6 & \textbf{85.3} & 3.2 & 92.6 \\
64K & 69.5 & \textbf{81.3} & 4.2 & 87.2 \\
128K & 65.7 & \textbf{76.8} & 3.0 & 83.0 \\
\midrule
Length mean & 73.0 & \textbf{84.4} & & \\
\bottomrule
\end{tabular}\par
\medskip
\raggedright
At 128K, variable tracking contributes about $80/13=6.15$ of the 11.1-point Overall gap. Overall weights all 13 tasks equally.
\papersource{Paper, Table 6, PDF p. 15; final 128K checkpoint. Contribution is a reader calculation.}
\end{frame}

% 16
\begin{frame}{Long-context gains vary by task and training stage}
\centering
\begin{tabular}{@{}lrrrr@{}}
\toprule
& \multicolumn{2}{c}{32K checkpoint} & \multicolumn{2}{c}{128K checkpoint} \\
Score $\uparrow$ & GQA & VHA & GQA & VHA \\
\midrule
LongBench-v2 & \textbf{28.0} & 22.3 & 18.9 & \textbf{24.2} \\
InfiniteBench & \textbf{29.9} & 27.3 & \textbf{31.8} & 31.5 \\
BABILong & 36.5 & \textbf{37.4} & 31.1 & \textbf{38.2} \\
Short eight-task mean & 62.7 & \textbf{63.7} & 62.6 & \textbf{64.1} \\
\bottomrule
\end{tabular}\par
\medskip
\raggedright
\begin{itemize}
\item LongBench-v2 ranking reverses between checkpoints.
\item Final HELMET length means: 3 wins / 1 tie / 1 loss.
\item HELMET ICL favors GQA at all five tested lengths.
\end{itemize}
\papersource{Paper, Tables 6--7, PDF p. 15. BPB and HELMET details are in the backup slides.}
\end{frame}

% 17
\begin{frame}{Small linear factors work best in the ablations}
\begin{columns}[T]
\column{0.47\textwidth}
\begin{block}{Postmix width, 1.7B}
\centering
\begin{tabular}{@{}rr@{}}
\toprule
$r$ & Avg. \\
\midrule
4 & \textbf{54.50} \\
32 & 54.24 \\
128 & 54.15 \\
\bottomrule
\end{tabular}\par
\end{block}
Training loss decreases with larger $r$, but downstream accuracy falls.
\column{0.49\textwidth}
\begin{block}{SiLU insertion, 1.7B}
\centering
\begin{tabular}{@{}lr@{}}
\toprule
Variant & Avg. \\
\midrule
Premix + SiLU & 53.83 \\
Postmix + SiLU & 53.79 \\
Both + SiLU & 53.67 \\
Linear default & \textbf{54.50} \\
\bottomrule
\end{tabular}\par
\end{block}
Nonlinearity also prevents exact projection absorption.
\end{columns}
\medskip
At $H_v=16$, factor widths 32 and 128 both allow rank at most 16.
\papersource{Paper, Sec. 4.4 and Table 4, PDF p. 8. Rank cap follows from matrix dimensions.}
\end{frame}

% 18
\begin{frame}{Premix learns different transforms for the two groups}
\centering
\includegraphics[width=0.98\linewidth,height=0.46\textheight,keepaspectratio]{figures/fig-02-premix.pdf}
\raggedright
\begin{itemize}
\item Left: distance from identity across layers; peak group mean 5.85.
\item Right: distance between the two group transforms; peak 6.13.
\item The trained factors are nontrivial and group-specific.
\end{itemize}
\papersource{Paper, Fig. 2, Sec. 5, PDF p. 9. Trained 1.7B model; the two peaks occur at different layers.}
\end{frame}

% 19
\begin{frame}{Postmix mainly redistributes within KV groups}
\centering
\includegraphics[width=0.98\linewidth,height=0.48\textheight,keepaspectratio]{figures/fig-03-postmix.pdf}
\raggedright
\begin{itemize}
\item Source heads flow to recombined heads; color denotes KV group.
\item Strong off-diagonal mixing, mainly within groups, with cross-group bridges.
\item Absolute residual weights omit signs and the identity path.
\end{itemize}
\papersource{Paper, Fig. 3, Sec. 5, PDF pp. 9--10. Trained 4B model. This is a weight-structure probe.}
\end{frame}

% 20
\begin{frame}{Repeated runs and initialization support the design}
\begin{columns}[T]
\column{0.48\textwidth}
\begin{block}{Repeated 0.6B training}
\centering
\begin{tabular}{@{}lrr@{}}
\toprule
Model & Runs & Mean $\pm$ SD \\
\midrule
VHA & 4 & $50.40\pm0.07$ \\
GQA & 3 & $50.00\pm0.34$ \\
\bottomrule
\end{tabular}\par
\end{block}
SD is the sample standard deviation across run averages.
\column{0.48\textwidth}
\begin{block}{Premix initialization at 1.7B}
\centering
\begin{tabular}{@{}lr@{}}
\toprule
Initialization & Avg. \\
\midrule
$I+0.1\epsilon$ & \textbf{54.04} \\
$I+1.0\epsilon$ & 53.45 \\
Fully random & 53.11 \\
\bottomrule
\end{tabular}\par
\end{block}
All three runs were stable; the default gave the highest score.
\end{columns}
\medskip
These studies use a different training implementation from the main tables; compare within each study.
\papersource{Paper, Appendix B, Tables 8--10, PDF pp. 14--17.}
\end{frame}

% 21
\begin{frame}{The evidence supports quality and arithmetic savings}
\begin{block}{Directly supported}
Exact algebraic absorption; smaller KV cache; reported FLOPs; quality gains under the evaluated training recipes.
\end{block}
\begin{itemize}
\item The claimed $2$--$4\times$ decoding speedup is a memory-bound estimate. No end-to-end serving benchmark is reported.
\item Standard-GQA absorption expands Q; factorized prefill savings require a compatible execution path.
\item Weight probes suggest specialization and communication; targeted interventions would test their causal roles.
\end{itemize}
\papersource{Paper, Sec. 3.5, p. 6; Sec. 5, pp. 9--10; inference analysis based on Algorithm 1, p. 20.}
\end{frame}

% 22
\begin{frame}{Train with structure; deploy with ordinary GQA}
\begin{enumerate}
\item \textbf{Share and specialize queries.}\\
Two group transforms expand a compact physical query basis.
\medskip
\item \textbf{Change the output parameterization.}\\
Low-rank residual mixing improves the tested training outcomes.
\medskip
\item \textbf{Keep deployment simple.}\\
Both factors merge into standard projections with two KV heads.
\end{enumerate}
\medskip
\alert{Discussion:} Does the benefit persist at equal training FLOPs, and which learned connections actually cause the gain?
\papersource{Xibo Ding et al., \emph{Virtual Head Attention}, arXiv:2610.17959v1, 2026.\\
\href{https://arxiv.org/abs/2610.17959v1}{https://arxiv.org/abs/2610.17959v1}. Next: technical backup slides.}
\end{frame}

\appendix
% 23
\begin{frame}{Backup: Premix couples gradients across heads and groups}
Let $D_{i,g}=\partial\mathcal L/\partial\widetilde Q_{i,g}$. Then
\[
\frac{\partial\mathcal L}{\partial W_Q^{(i)}}
=\sum_{g=1}^{H_{kv}}X^\top D_{i,g}
\bigl(\prem\bigr)^\top,
\]
\[
\frac{\partial\mathcal L}{\partial\prem}
=\sum_{i=1}^{H_q'}(XW_Q^{(i)})^\top D_{i,g}.
\]
\begin{itemize}
\item A physical query basis aggregates signals from both KV groups.
\item A group transform aggregates signals from all physical heads.
\item This establishes different update dynamics; the downstream benefit is empirical.
\end{itemize}
\papersource{Paper, Appendix G, Eqs. (12)--(13), PDF p. 18.}
\end{frame}

% 24
\begin{frame}{Backup: the transpose comes from row-major flattening}
For $Y\in\R^{H_v\times d}$ and $M=I+BA^\top$,
\[
\rowflat(MY)=\rowflat(Y)(M^\top\otimes I_d).
\]
Thus
\begin{align*}
\rowflat(Y')W_O
&=\rowflat(Y)\bigl((I+AB^\top)\otimes I_d\bigr)W_O\\
&=\rowflat(Y)W_O^{\mathrm{abs}}.
\end{align*}
\begin{block}{What is required?}
Consistent head ordering and adjacent linear operations.\\
$M$ need not be invertible for this absorption identity.
\end{block}
This preserves real-arithmetic outputs; the deployed Q projection contains all virtual heads.
\papersource{Paper, Appendix H, Eqs. (14)--(18), PDF p. 20. Invertibility observation is a reader clarification.}
\end{frame}

% 25
\begin{frame}{Backup: applying a shared matrix still costs FLOPs}
Per token and layer, counting a multiply-add as two FLOPs:
\[
\begin{aligned}
F_Q^{\mathrm{GQA}} &=2hH_vd,\\
F_Q^{\mathrm{VHA}} &=2hH_q'd+2H_{kv}H_q'd^2,\\
F_{\mathrm{Postmix}} &=4H_vrd.
\end{aligned}
\]
\begin{itemize}
\item At 1.7B: $h=2048,d=128,H_q'=8,H_v=16,r=4$.
\item Q FLOPs: $8{,}388{,}608\to4{,}194{,}304+524{,}288$.
\item Postmix adds $32{,}768$ FLOPs.
\item Shared Premix parameters are counted once, but applied to every physical head.
\end{itemize}
\papersource{Reader calculation from Sec. 3.5, PDF p. 5, and Appendix G, p. 18. These are forward costs, not total training FLOPs.}
\end{frame}

% 26
\begin{frame}{Backup: joint resource trade-offs across all scales}
\centering
\begin{tabular}{@{}lrrrrr@{}}
\toprule
& \multicolumn{3}{c}{Training G/token, $S=4096$} & \multicolumn{2}{c}{KV KB/layer/token} \\
Scale & GQA & MFA & VHA & GQA & VHA \\
\midrule
0.6B & 7.57 & 11.54 & 6.96 & 4.00 & 1.00 \\
1.7B & 10.61 & 15.90 & 9.78 & 4.00 & 1.00 \\
4B & 24.38 & 36.04 & 22.52 & 4.00 & 1.00 \\
8B-A1B & 10.27 & 17.82 & 9.60 & 2.00 & 1.00 \\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Reported VHA training reductions: 8.1 / 7.9 / 7.6 / 6.5\%.
\item MFA matches the 1 KB cache but uses larger head dimension.
\item MHA uses fewer query heads in the matched configurations; it can have fewer FLOPs but a much larger cache.
\end{itemize}
\papersource{Paper, Table 5, PDF p. 8. Relative percentages are copied from the paper; displayed FLOPs are rounded.}
\end{frame}

% 27
\begin{frame}{Backup: final HELMET and language-modeling results}
\begin{columns}[T]
\column{0.49\textwidth}
\textbf{HELMET length means} $\uparrow$
\medskip

\begin{tabular}{@{}lrr@{}}
\toprule
Category & GQA & VHA \\
\midrule
RAG & 28.14 & \textbf{37.48} \\
LongQA & 15.70 & \textbf{16.10} \\
Summarization & 6.94 & \textbf{7.20} \\
ICL & \textbf{72.68} & 68.20 \\
Reranking & 0.84 & 0.84 \\
\bottomrule
\end{tabular}\par
\column{0.47\textwidth}
\textbf{ProLong bits per byte} $\downarrow$
\medskip

\begin{tabular}{@{}lrr@{}}
\toprule
Domain & GQA & VHA \\
\midrule
arXiv & 0.582 & \textbf{0.564} \\
Books & 0.819 & \textbf{0.804} \\
Code & 0.311 & \textbf{0.296} \\
Web & 0.768 & \textbf{0.760} \\
\bottomrule
\end{tabular}\par
\end{columns}
\bigskip
HELMET means are reader calculations over 8K, 16K, 32K, 64K, and 128K, using the final checkpoint.
\papersource{Paper, Tables 6--7, PDF p. 15. BPB uses the final 128K checkpoint.}
\end{frame}

% 28
\begin{frame}{Backup: category gains hide meaningful task differences}
\centering
\begin{tabular}{@{}lrrr@{}}
\toprule
30B-A3B benchmark & GQA & MLA & VHA \\
\midrule
MMLU & 55.97 & 59.34 & \textbf{60.24} \\
MMLU-Pro & 36.20 & \textbf{41.12} & 40.03 \\
GPQA & \textbf{34.62} & 31.62 & 34.11 \\
GSM8K & \textbf{56.48} & 55.42 & 56.25 \\
MATH & 38.18 & 43.42 & \textbf{48.84} \\
HumanEval+ & 50.00 & \textbf{52.44} & 47.56 \\
\bottomrule
\end{tabular}\par
\medskip
\raggedright
Four categories contain 4 / 3 / 3 / 2 benchmarks. The reported overall score is
\[
\frac14(\overline s_{\mathrm{General}}+
\overline s_{\mathrm{Math}}+
\overline s_{\mathrm{Coding}}+
\overline s_{\mathrm{Multilingual}}).
\]
\papersource{Paper, Tables 3 and 11, PDF pp. 7 and 17. Each benchmark retains the paper's evaluation protocol.}
\end{frame}

% 29
\begin{frame}{Backup: model sizes share a consistent optimizer recipe}
\centering
\begin{tabular}{@{}lrrrrr@{}}
\toprule
 & 0.6B & 1.7B & 4B & 8B-A1B & 30B-A3B \\
\midrule
Layers & 28 & 28 & 36 & 24 & 46 \\
Hidden width & 1024 & 2048 & 2560 & 2048 & 1536 \\
VHA physical Q & 8 & 8 & 16 & 16 & 24 \\
VHA virtual Q & 16 & 16 & 32 & 32 & 48 \\
Postmix $r$ & 4 & 4 & 8 & 8 & 12 \\
Training tokens & 50B & 50B & 50B & 50B & 500B \\
Context & 4K & 4K & 4K & 4K & 8K \\
\bottomrule
\end{tabular}\par
\medskip
\raggedright
AdamW: $(\beta_1,\beta_2)=(0.9,0.95)$; weight decay 0.1;\\
gradient clipping 1.0; warmup 2000 steps.\\
Premix: $\alpha=0.1$, $\sigma=1$; two KV heads throughout.
\papersource{Paper, Tables 1, 3 and 12, PDF pp. 6--7, 18. The separate 10B long-context recipe is on slide 14.}
\end{frame}

% 30
\begin{frame}{Backup: VHA loss is competitive across four scales}
\centering
\includegraphics[width=0.95\linewidth,height=0.72\textheight,keepaspectratio]{figures/fig-04-loss.pdf}
\papersource{Paper, Fig. 4, PDF p. 19. 50B tokens; 4K context; EMA smoothing $\alpha=0.93$; curves start at step 4K.\\
VHA (solid blue) is lowest at 4B; the paper reports a gap within 0.01 nats from the best at the other scales.}
\end{frame}

\end{document}