\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\la}{\mathrm{la}}
\newcommand{\nla}{\mathrm{nla}}
\newcommand{\Norm}{\operatorname{RMSNorm}}
\newcommand{\softmax}{\operatorname{softmax}}
\newcommand{\fig}[3]{\includegraphics[width=#1,height=#2,keepaspectratio]{figures/#3}}

\title[NAtS-L]{Neural Attention Search Linear}
\subtitle{Towards Adaptive Token-Level Hybrid Attention Models}
\author{Paper authors: Difan Deng, Andreas Bentzen Winje,\\Lukas Fehring, and Marius Lindauer}
\institute{Leibniz University Hannover / L3S\\\medskip Independent paper discussion}
\date{arXiv:2602.03681v2 --- revised June 2, 2026\\Reading date: October 15, 2026}

\begin{document}

\begin{frame}
\titlepage
\vspace{-4mm}
\begin{center}\small Learn which historical blocks need explicit KV memory.\end{center}
\end{frame}

\begin{frame}{Two memory types offer different trade-offs}
\centering
\fig{12.5cm}{3.35cm}{fig-01-attention-maps.pdf}
\begin{columns}[T,onlytextwidth]
\column{0.31\textwidth}\textbf{Softmax}\par Explicit historical KV.\par Expensive global reads.
\column{0.31\textwidth}\textbf{GDN}\par Fixed matrix state.\par Compression can lose detail.
\column{0.34\textwidth}\textbf{NAtS-L}\par Retain selected KV blocks.\par Compress the others.
\end{columns}
\papersource{Paper, Fig. 1, PDF p. 2; Secs. 1--3, pp. 1--4. Blue: softmax; green: linear state; cyan: intra-chunk operations.}
\end{frame}

\begin{frame}{Route memory writes; mix reads for every query}
\begin{block}{The routing unit is a 64-token block within each head group}
A block is assigned to explicit softmax KV or to the GDN state.
\end{block}
\medskip
\begin{tabular}{>{\raggedright\arraybackslash}p{3.1cm}>{\raggedright\arraybackslash}p{4.5cm}>{\raggedright\arraybackslash}p{5.0cm}}
\toprule
Decision & Input signal & Effect \\
\midrule
Storage route & Mean-pooled block features & Select the historical KV columns \\[2mm]
Readout mixture & Current query vector & Weight both normalized outputs \\
\bottomrule
\end{tabular}\par
\medskip
\alert{All queries can read both memories.} The route does not select a single query-side expert.
\papersource{Paper, Secs. 4.1 and 4.3, Eq. (8)--(14), (21)--(22), PDF pp. 4--6.}
\end{frame}

\begin{frame}{Shared projections feed two different operators}
\begin{columns}[c,onlytextwidth]
\column{0.52\textwidth}\centering
\fig{7.1cm}{5.2cm}{fig-02-architecture.pdf}
\column{0.45\textwidth}
\begin{itemize}
\item Shared $Q,K,V$ projections: linear, short convolution, SiLU.
\item Softmax $Q,K$: RMS normalization.
\item GDN $Q,K$: L2 normalization.
\item Normalize each output, mix, gate, and project.
\end{itemize}
\end{columns}
\papersource{Paper, Fig. 2 and Sec. 4.3, PDF pp. 5--6. Figure cropped from the paper.}
\end{frame}

\begin{frame}{A small router selects historical KV blocks}
\[
\operatorname{score}_t=W^{\operatorname{score}}\operatorname{Mean}(X_{[t]}),
\qquad \operatorname{opt}_t=\arg\max_a\softmax(\operatorname{score}_t)_a
\]
\begin{itemize}
\item $X_{[t]}\in\mathbb R^{C\times d}$; two scores per block and head group.
\item Softmax loads selected historical KV; GDN writes selected blocks.
\item 800M setup: 6 GDN heads, 18 softmax heads; each group of 3 softmax heads shares a routing mask.
\end{itemize}
\begin{block}{Training objective}
Language modeling loss only: no fixed softmax quota or balancing loss.
\end{block}
\papersource{Paper, Eq. (8)--(14), Sec. 4.3, Appendix A.1, PDF pp. 4--6, 15.}
\end{frame}

\begin{frame}{Commit the route only after the block is complete}
\begin{enumerate}
\item At a block boundary, save the incoming GDN state.
\item Within the current block, compute \textbf{both causal branches}: temporarily update the state and cache the new KV.
\item Once the entire block is visible, select its route for future queries.
\end{enumerate}
\medskip
\begin{columns}[T,onlytextwidth]
\column{0.48\textwidth}
\begin{block}{Selected: GDN}
Keep the state update.\par Discard this block's temporary KV.
\end{block}
\column{0.48\textwidth}
\begin{block}{Selected: softmax}
Keep the KV. Undo GDN content writes, while retaining decay.
\end{block}
\end{columns}
\papersource{Paper, Sec. 4.1, PDF p. 4; GDN-specific decay in Eq. (20), p. 5. Process summarized for teaching.}
\end{frame}

\begin{frame}{GDN corrects memory and keeps its clock running}
\[
S_i=\alpha_iS_{i-1}(I-\beta_i k_i k_i^\top)+\beta_i v_i k_i^\top
\]
\[
\underbrace{S_i=\alpha_iS_{i-1}
+\beta_i(v_i-\alpha_iS_{i-1}k_i)k_i^\top}_{\text{Equivalent expansion: decay, then a prediction-error write}}
\]
\begin{itemize}
\item $S_i\in\mathbb R^{d_v\times d_k}$; $\alpha_i$ controls forgetting, $\beta_i$ the write strength.
\item For a softmax-selected block, no GDN content write is committed:
\end{itemize}
\[
S_{[t+1]}=\left(\prod_{i\in[t]}\alpha_i\right)S_{[t]}.
\]
\papersource{Paper, Eq. (19)--(20), PDF p. 5. Middle equation is an algebraic teaching expansion.}
\end{frame}

\begin{frame}{Normalize branch scales before query-dependent mixing}
\[
O_i=w_i^{\nla}\Norm(O_i^{\nla})+w_i^{\la}\Norm(O_i^{\la})
\]
\[
\begin{bmatrix}w_i^{\nla}\\w_i^{\la}\end{bmatrix}
=\softmax\!\left(\begin{bmatrix}W^{\nla}\\W^{\la}\end{bmatrix}q_i\right)
\]
\begin{itemize}
\item Softmax and GDN outputs have different scale behavior.
\item Separate RMSNorm makes their combination easier to control.
\item The current query sets nonnegative, sum-to-one weights per head.
\end{itemize}
\begin{block}{Two distinct control signals}
Block features decide \emph{what to retain}; queries decide \emph{how to read}.
\end{block}
\papersource{Paper, Eq. (21)--(22), PDF p. 6.}
\end{frame}

\begin{frame}{Sparse training skips inactive routing gradients}
For a continuously relaxed softmax mask, a useful interpretation is
\[
\frac{\partial\mathcal L}{\partial M_{ij}}
=P_{ij}\langle g_i,V_j-O_i\rangle,
\quad P_{ij}=\frac{e^{A_{ij}}}{\sum_k e^{A_{ik}}M_{ik}},
\]
where $g_i=\partial\mathcal L/\partial O_i$. Aggregate columns within each block.
\medskip
\begin{itemize}
\item Evaluating inactive softmax choices would revisit all KV pairs.
\item The method sets inactive score gradients to zero to preserve sparse cost.
\item The paper does not fully specify the surrogate chain through hard $\arg\max$.
\end{itemize}
\papersource{Paper, Sec. 4.2, Eq. (15)--(17), PDF pp. 4--5. Equation above is a continuous-mask teaching derivation.}
\end{frame}

\begin{frame}{The softmax term can still grow quadratically}
The paper suppresses fixed width and chunk factors:
\[
\mathcal O(LL_{\nla}+L_{\la}),\qquad
\rho=\frac{L_{\nla}}{L}\ \Longrightarrow\ \mathcal O(\rho L^2+L).
\]
\begin{itemize}
\item A constant retained fraction reduces the coefficient, not the order.
\item No budget constraint guarantees a fixed number of retained KV.
\item Every query still reads the linear state and computes local outputs.
\end{itemize}
\medskip
Illustrative single-head memory, with width $d_h$:
\[
\underbrace{\mathcal O(d_h^2)}_{\text{matrix state}}+
\underbrace{\mathcal O((L_{\nla}+C)d_h)}_{\text{retained KV + current block}}.
\]
\papersource{Paper, Sec. 4.1, PDF p. 4. Fraction substitution and memory decomposition are this presentation's analysis.}
\end{frame}

\begin{frame}{The strongest variant mixes both across and within layers}
\centering
\begin{tabular}{lr@{\hspace{5mm}}l}
\toprule
Model & Parameters & Layer composition \\
\midrule
GDN & 793M & 21 GDN \\
Mamba2 & 801M & 48 Mamba2 \\
Transformer & 778M & 24 softmax \\
GDN Hybrid & 802M & 17 GDN + 5 softmax \\
NAtS-L & 794M & 21 NAtS-L \\
\textbf{NAtS-L Hybrid} & 793M & 15 GDN + 6 NAtS-L \\
\bottomrule
\end{tabular}\par
\medskip
\begin{block}{A material configuration difference}
NAtS-L uses RoPE in its softmax branch. NAtS-L Hybrid uses no positional encoding in that branch.
\end{block}
\papersource{Paper, Sec. 5, PDF p. 6. Exact layer counts shown instead of the approximate ``3:1'' description.}
\end{frame}

\begin{frame}{Training length is fixed; evaluation probes distinct skills}
\begin{block}{Pretraining}
FineWeb-Edu; 4,096-token context; 380M / 15B tokens and 800M / 50B tokens. AdamW; 0.5M tokens per effective batch.
\end{block}
\medskip
\begin{tabular}{p{4.3cm}p{8.5cm}}
\toprule
Evaluation & What it measures \\
\midrule
Commonsense / short PPL & General modeling quality \\
Six retrieval tasks & Read information from contexts truncated to 4,096 \\
Long-text PPL & Modeling beyond the training length, up to 65,536 \\
RULER retrieval subset & Retrieval at 4K, 8K, and 16K \\
LongBench-e subsets & QA, summarization, classification, and code \\
Latency & Prefill and decode with inputs up to 128K \\
\bottomrule
\end{tabular}\par
\papersource{Paper, Sec. 5.1 and Appendix A.1, PDF pp. 6--8, 15. Latency length is not a retrieval-quality result.}
\end{frame}

\begin{frame}{Retrieval improves more than short-context reasoning}
\centering
\begin{tabular}{lrrr}
\toprule
800M model & Wiki PPL $\downarrow$ & Commonsense $\uparrow$ & Retrieval $\uparrow$ \\
\midrule
GDN & 19.54 & 51.02 & 31.22 \\
Mamba2 & 19.28 & 51.38 & 29.94 \\
Transformer & 19.27 & 50.78 & 33.66 \\
GDN Hybrid & 18.83 & \textbf{51.52} & 37.30 \\
NAtS-L & 19.17 & 50.84 & 39.75 \\
NAtS-L Hybrid & \textbf{18.52} & 51.41 & \textbf{45.07} \\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Retrieval gain over GDN Hybrid: \alert{+7.77 percentage points}.
\item Hybrid wins 5/6 retrieval tasks; FDA reaches 67.70 vs. 31.13.
\item DROP favors NAtS-L: 22.90 vs. Hybrid's 20.84.
\end{itemize}
\papersource{Paper, Table 1, PDF p. 8. Means: 7 commonsense tasks and 6 retrieval tasks; retrieval context capped at 4,096.}
\end{frame}

\begin{frame}{RULER extrapolation improves but still degrades}
\centering
\begin{tabular}{lrrr}
\toprule
800M model & 4K & 8K & 16K \\
\midrule
GDN & 0.25 & 0.14 & 0.04 \\
Mamba2 & 0.18 & 0.09 & 0.04 \\
Transformer & 0.45 & 0.00 & 0.00 \\
GDN Hybrid & 0.47 & 0.02 & 0.00 \\
NAtS-L & 0.39 & 0.13 & 0.05 \\
NAtS-L Hybrid & \textbf{0.49} & \textbf{0.32} & \textbf{0.21} \\
\bottomrule
\end{tabular}\par
\medskip
Hybrid is best at each length, yet drops from 0.49 to 0.21.\par
Single-needle retrieval is a clearer strength than harder multi-item tasks.
\papersource{Paper, Table 2, PDF p. 8; per-task Fig. 9, p. 17. RULER retrieval subset; mean score on a 0--1 scale, higher is better.}
\end{frame}

\begin{frame}{Longer narrative context keeps helping the Hybrid}
\begin{columns}[c,onlytextwidth]
\column{0.49\textwidth}\centering
\fig{7cm}{3.8cm}{ppl-codeparrot.png}
\column{0.49\textwidth}\centering
\fig{7cm}{3.8cm}{ppl-narrativeqa.png}
\end{columns}
\centering\fig{12.4cm}{0.55cm}{ppl-legend.png}
\medskip
\raggedright
\begin{itemize}
\item NarrativeQA PPL approaches 15 near 64K; training length was 4K.
\item CodeParrot uses concatenated shorter examples.
\item RoPE and layer layout differ across variants, alongside routing.
\end{itemize}
\papersource{Paper, two panels of Fig. 3, PDF p. 8; interpretation p. 7. Axes: context length and per-token PPL, lower is better.}
\end{frame}

\begin{frame}{LongBench gains depend on the task}
\centering
\begin{tabular}{lrrl}
\toprule
Below 4K, 800M & GDN Hybrid & NAtS-L Hybrid & Best model \\
\midrule
GovReport & 12.54 & \textbf{17.26} & NAtS-L Hybrid \\
TriviaQA & 20.10 & \textbf{29.79} & NAtS-L Hybrid \\
SAMSum & 16.21 & \textbf{24.98} & NAtS-L Hybrid \\
Qasper & 4.28 & 2.40 & Mamba2: 4.79 \\
TREC & \textbf{41.00} & 17.00 & GDN Hybrid \\
LCC & 20.11 & 9.43 & Mamba2: 20.68 \\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Hybrid wins 5/11 tasks overall; it also wins HotpotQA and MultiFieldQA.
\item QA and some summarization tasks benefit; code is not a uniform strength.
\end{itemize}
\papersource{Paper, Table 3, PDF p. 9. Selected task columns; task-specific scores, all higher is better.}
\end{frame}

\begin{frame}{Long inputs amortize the extra inference machinery}
\begin{columns}[c,onlytextwidth]
\column{0.65\textwidth}\centering
\fig{9.2cm}{4.9cm}{fig-04-latency.png}
\column{0.32\textwidth}
\textbf{At 128K input}\par\medskip
Prefill: \alert{5.4$\times$} faster than Transformer.\par\medskip
Still \textbf{1.66$\times$} GDN's prefill latency.\par\medskip
Decode: \alert{2.3$\times$} faster than Transformer.
\end{columns}
\medskip
At short inputs, Hybrid decoding is slower in the plotted comparison.
\papersource{Paper, Fig. 4, PDF p. 8; ratios reported in Sec. 5.1, p. 7. Prefill: seconds; decode: milliseconds. Full timing protocol is not specified.}
\end{frame}

\begin{frame}{Routing adapts to the dataset, layer, and head group}
\centering
\fig{12.3cm}{4.6cm}{fig-05-head-routing.png}
\par\smallskip
\raggedright
Text datasets look similar; code changes some head assignments.\par
Several head groups are almost entirely linear; others mix memory types.
\papersource{Paper, Fig. 5, PDF p. 8: NAtS-L Hybrid. Columns: datasets; rows: layers; x-axis: 6 head groups; bar height: softmax fraction.}
\end{frame}

\begin{frame}{Ablations support the complete two-path design}
\centering
\begin{tabular}{lrr}
\toprule
Variant & Retrieval mean $\uparrow$ & Change \\
\midrule
Default Hybrid & \textbf{33.49} & --- \\
Only softmax intra-chunk output & 30.87 & $-2.62$ \\
Only GDN intra-chunk output & 26.98 & $-6.51$ \\
No decay across softmax blocks & 32.24 & $-1.25$ \\
Normalize only after summation & 29.59 & $-3.90$ \\
Weights from $X$ instead of $q$ & 27.64 & $-5.85$ \\
Chunk size 128 instead of 64 & 27.85 & $-5.64$ \\
\bottomrule
\end{tabular}\par
\medskip
Query-conditioned readout and sufficiently fine block routing both matter.
\papersource{Paper, Table 4, PDF p. 9. Baseline matches the 380M results in Table 6; it is not the 800M mean of 45.07. Changes computed from table values.}
\end{frame}

\begin{frame}{The useful idea is selective explicit memory}
\begin{enumerate}
\item \textbf{Mechanism:} choose storage per block, then read both memories with query-dependent weights.
\item \textbf{Evidence:} Hybrid improves retrieval and length extrapolation, with long-input latency gains.
\item \textbf{Trade-off:} no fixed KV budget; dual intra-chunk work; task-specific quality differences.
\end{enumerate}
\begin{block}{Most informative next comparisons}
Control positional encoding and layer layout; sweep KV budgets and chunk size; compare quality at matched latency and memory.
\end{block}
\papersource{Synthesis of Secs. 4--6. Suggested comparisons are this presentation's analysis.}
\end{frame}

\begin{frame}{Paper and discussion guide}
\textbf{Deng, Winje, Fehring, and Lindauer (2026).}\par
Neural Attention Search Linear: Towards Adaptive Token-Level Hybrid Attention Models.\par\medskip
\href{https://arxiv.org/abs/2602.03681v2}{arXiv:2602.03681v2}\par
\href{https://github.com/automl/NeuralAttentionSearchLinear}{Author code repository linked by the paper}
\medskip
\begin{itemize}
\item Method: Sec. 4, PDF pp. 4--6.
\item Main evidence: Tables 1--4 and Figures 3--6, pp. 8--9.
\item Additional evidence: Appendices A--C, pp. 15--21.
\end{itemize}
\medskip
Main discussion: approximately 25 minutes. Backup slides follow.
\end{frame}

\appendix

\begin{frame}{Backup: training and head configurations}
\centering
\begin{tabular}{lrr}
\toprule
Setting & 380M scale & 800M scale \\
\midrule
Training tokens & 15B & 50B \\
Context length & 4,096 & 4,096 \\
Effective batch tokens & 0.5M & 0.5M \\
Model width & 1,024 & 1,536 \\
Warmup tokens & 0.5B & 1B \\
GDN heads & 6 & 6 \\
NAtS-L softmax heads & 12 & 18 \\
Softmax heads per routing group & 2 & 3 \\
\bottomrule
\end{tabular}\par
\medskip
AdamW; peak LR $3\!\times\!10^{-4}$, initial/final LR $3\!\times\!10^{-5}$; cosine schedule. Short-convolution kernel: 4. Default chunk size: 64.
\papersource{Paper, Sec. 5 and Appendix A.1, PDF pp. 6, 15. Training: 4 or 8 H100 PCIe 80GB GPUs.}
\end{frame}

\begin{frame}{Backup: normalized query weights outperform alternatives}
\centering
\begin{tabular}{lrr}
\toprule
Variant & Retrieval mean $\uparrow$ & Change \\
\midrule
Default: softmax weights from $q$ & \textbf{33.49} & --- \\
No attention output weights & 32.34 & $-1.15$ \\
Sigmoid weights & 30.02 & $-3.47$ \\
Linear weights, no activation & 30.57 & $-2.92$ \\
Weights from $X$ & 27.64 & $-5.85$ \\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Separate branch normalization and normalized mixing solve different problems.
\item The best mean does not imply winning each retrieval task.
\item Intra-chunk ablations change local terms, not the entire historical branch.
\end{itemize}
\papersource{Paper, Table 4, PDF p. 9. Changes computed from published means.}
\end{frame}

\begin{frame}{Backup: sparse baselines use a different budget protocol}
\begin{columns}[T,onlytextwidth]
\column{0.53\textwidth}
\begin{tabular}{lr}
\toprule
Table 5, 380M & Mean $\uparrow$ \\
\midrule
Transformer & 22.99 \\
NSA / MOBA & 22.05 / 21.98 \\
GDN SWA & 8.53 \\
FlexPrefill & 20.71 \\
SnapKV & 18.64 \\
AdaKV / PyramidKV & 17.96 / 18.41 \\
NAtS-L & 28.48 \\
NAtS-L Hybrid & \textbf{34.19} \\
\bottomrule
\end{tabular}\par
\column{0.44\textwidth}
Sparse variants use 50\% of the stated computational budget. NAtS-L has no fixed routing fraction.\par\medskip
\textbf{Reporting discrepancy:}\par
Hybrid NQ is 20.72 here, but 16.53 in Table 6. Means are 34.19 and 33.49, respectively.\par\medskip
Keep comparisons within each table.
\end{columns}
\papersource{Paper, Appendix B and Table 5, PDF pp. 15--16; Table 6, p. 17. The NQ discrepancy is not explained in the paper.}
\end{frame}

\begin{frame}{Backup: smaller models preserve the retrieval trend}
\centering
\begin{tabular}{lrrrr}
\toprule
380M / 15B tokens & Retrieval $\uparrow$ & RULER 4K & 8K & 16K \\
\midrule
GDN & 20.78 & 0.15 & 0.07 & 0.02 \\
Mamba2 & 19.25 & 0.09 & 0.03 & 0.02 \\
Transformer & 21.71 & 0.19 & 0.00 & 0.00 \\
GDN Hybrid & 24.97 & 0.21 & 0.07 & 0.00 \\
NAtS-L & 27.16 & 0.19 & 0.08 & 0.03 \\
NAtS-L Hybrid & \textbf{33.49} & \textbf{0.36} & \textbf{0.30} & \textbf{0.12} \\
\bottomrule
\end{tabular}\par
\medskip
Commonsense remains close: Hybrid 43.04; Transformer 43.26.\par
The winning LongBench tasks differ from those at 800M scale.
\papersource{Paper, Tables 6--8, PDF pp. 17--18. Retrieval: percent-scale mean; RULER: 0--1 retrieval-subset mean.}
\end{frame}

\begin{frame}{Backup: Mamba2 can also supply the compressed memory}
\centering
\begin{tabular}{lrr}
\toprule
Retrieval task & Mamba2 & NAtS-L Mamba2 \\
\midrule
SWDE & 13.77 & \textbf{32.58} \\
SQD & \textbf{27.92} & 21.18 \\
FDA & 4.26 & \textbf{32.30} \\
TQA & \textbf{44.91} & 41.35 \\
NQ & 9.06 & \textbf{12.48} \\
DROP & \textbf{15.57} & 12.12 \\
\midrule
Mean & 19.25 & \textbf{25.34} \\
\bottomrule
\end{tabular}\par
\medskip
Mean gain: +6.09 percentage points, with losses on three tasks.\par
This tests a different recurrent backbone, not a multi-operator search space.
\papersource{Paper, Appendix C.2 and Table 9, PDF pp. 17--18. All task scores higher is better.}
\end{frame}

\begin{frame}{Backup: longer LongBench contexts retain task trade-offs}
\centering
\begin{tabular}{llrr}
\toprule
Context group & Task & Hybrid & Best other model \\
\midrule
Table 10 & TriviaQA & \textbf{32.17} & GDN: 22.43 \\
 & GovReport & \textbf{13.46} & NAtS-L: 8.07 \\
 & LCC & 10.28 & \textbf{Mamba2: 21.68} \\
\midrule
Beyond 8K & TriviaQA & \textbf{28.60} & NAtS-L: 23.29 \\
 & GovReport & \textbf{9.10} & NAtS-L: 7.92 \\
 & LCC & 13.31 & \textbf{Mamba2: 18.85} \\
\bottomrule
\end{tabular}\par
\medskip
Table 10 is presented as the next bin after 4K, but its caption only says ``below 8k.'' Table 11 is a $>8$K group, not a fixed 16K test.
\papersource{Paper, Appendix C.4, Tables 10--11, PDF pp. 18--19. 800M models, selected task-specific scores, higher is better.}
\end{frame}

\begin{frame}{Backup: routing fractions change over time}
\centering
\fig{12.1cm}{4.6cm}{fig-06-time-routing.png}
\par\smallskip
\raggedright
Each cell is a head-group/layer pair; time runs horizontally.\par
Blue: softmax fraction. Orange: GDN fraction. Layer 8, head 1 becomes more softmax-heavy.
\papersource{Paper, Fig. 6, PDF p. 9. This is storage allocation; query-side output weights are a different quantity (Fig. 12, p. 21).}
\end{frame}

\begin{frame}{Backup: the mask gradient includes renormalization}
For a fixed query, let
\[
Z_i=\sum_j e^{A_{ij}}M_{ij},\quad
O_i=\frac{\sum_j e^{A_{ij}}M_{ij}V_j}{Z_i}.
\]
Differentiating the numerator and denominator gives
\[
\frac{\partial O_i}{\partial M_{ij}}
=\frac{e^{A_{ij}}}{Z_i}(V_j-O_i),\qquad
\frac{\partial\mathcal L}{\partial M_{ij}}
=P_{ij}\langle g_i,V_j-O_i\rangle.
\]
\begin{itemize}
\item The subtraction of $O_i$ comes from normalization.
\item Sum over query positions and the KV columns of the same block.
\item This differentiates a continuous mask; it does not resolve hard routing by itself.
\end{itemize}
\papersource{Teaching derivation of the signal underlying Eq. (15)--(16), PDF p. 5. Inactive mask-score gradients are omitted by the method.}
\end{frame}

\end{document}