% Research reading slides, based on the arxiv-paper-reader Beamer template.
% Rebuild: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hspace{8mm}\scriptsize SSE $\cdot$ arXiv v2\hfill\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\SSE}{\textsc{SSE}}
\newcommand{\TT}{\mathcal T}
\newcommand{\myfig}[3]{\includegraphics[width=#2\linewidth,height=#3\textheight,keepaspectratio]{figures/#1.pdf}}
\title{Scaling Linear Attention with Sparse State Expansion}
\author{Research paper discussion}
\date{2 November 2026}
\hypersetup{pdftitle={SSE: Scaling Linear Attention with Sparse State Expansion -- reading slides},pdfauthor={Research paper discussion}}

\begin{document}

\begin{frame}\relax
 {\usebeamerfont{title}\color{AcademicBlue}Scaling Linear Attention with\par Sparse State Expansion\par}
 \vspace{3mm}
 {\small Paper by Yuqi Pan, Yongqi An, Zheng Li, Yuhong Chou, Ruijie Zhu,\par
 Xiaohui Wang, Mingxuan Wang, Jinqiao Wang, and Guoqi Li}
 \vspace{2mm}
 \begin{block}{The idea}
 Increase recurrent memory capacity with shared projections,\par
 and update only the partitions selected by each token.
 \end{block}
 {\small\href{https://arxiv.org/abs/2507.16577v2}{arXiv:2507.16577v2} (1 Oct 2025) $\cdot$ Reading: 2 Nov 2026\par
 Published at \href{https://proceedings.iclr.cc/paper_files/paper/2026/hash/d3b8ce5e27b1c622d1b3da22b215e59b-Abstract-Conference.html}{ICLR 2026} with ``Capacity'' added to the title.\par
 \href{https://proceedings.iclr.cc/paper_files/paper/2026/file/d3b8ce5e27b1c622d1b3da22b215e59b-Supplemental-Conference.zip}{Official code: conference supplement ZIP} (model and kernels; external dependencies).}
 \papersource{A 20--30 minute research discussion. Equations, figures, and results use arXiv v2.}
\end{frame}

\begin{frame}{The bottleneck is compressed memory capacity}
 \begin{center}
 \begin{tabular}{lcc}
 \toprule
 & Softmax attention & Linear attention \\
 \midrule
 Context state & $\{K_t,V_t\}$ & $S_t\in\mathbb R^{c\times d}$ \\
 State elements / head & $2td$ & $cd$ \\
 Per-token state work & grows with $t$ & fixed in $t$ \\
 Main compromise & cache and access cost & lossy history compression \\
 \bottomrule
 \end{tabular}
 \end{center}
 \vspace{2mm}
 \begin{itemize}
 \item $c=128$: the same element count as a 64-token KV cache.
 \item More model parameters and more context state are distinct resources.
 \item Goal: enlarge state capacity with little projection-parameter growth.
 \end{itemize}
 \papersource{Secs. 2--4, PDF pp. 3--6. The 64-token comparison is a storage count, not equal recall capacity.}
\end{frame}

\begin{frame}{A key vector decides where information is written}
 \[
 S_t=\Lambda_t S_{t-1}+\phi(k_t)^\top v_t
 \]
 \begin{columns}[T]
 \column{0.49\textwidth}
 \begin{block}{Manage history}
 $\Lambda_t$: decay, retention, or erasure.\par
 GLA uses diagonal gating;\par GDN uses a delta-rule transition.
 \end{block}
 \column{0.49\textwidth}
 \begin{block}{Allocate new information}
 $\phi(k_t)$: weights over state rows.\par
 Interpret the key projection as a learned classifier of inputs.
 \end{block}
 \end{columns}
 \vspace{2mm}
 A row is a learned feature bucket. Dense writes mix unrelated inputs;\par
 dense decay advances every bucket even when it is irrelevant.
 \papersource{Sec. 3, Eq. (6), PDF pp. 4--6. SSE experiments use GLA-style transitions.}
\end{frame}

\begin{frame}{SSE routes tokens into shared-parameter state partitions}
 \begin{columns}[c]
 \column{0.64\textwidth}
 \centering\myfig{fig-04-sse-arch}{1}{0.76}
 \column{0.34\textwidth}
 \begin{enumerate}
 \item Hard top-$k$ selection among $N$ partitions.
 \item Softmax allocation among rows inside each selected partition.
 \item One always-active shared partition.
 \end{enumerate}
 \vspace{2mm}
 \alert{Inactive partitions keep their state unchanged.}
 \end{columns}
 \papersource{Sec. 4.1, Fig. 4, PDF p. 7. Original author figure.}
\end{frame}

\begin{frame}{One gate controls both writes and reads}
 \[
 e_t=\operatorname{softmax}(x_tW_e),\quad
 \TT_t=\operatorname{TopKIndices}(e_t,k)
 \]
 \[
 q_t=x_tW_q,\qquad v_t=x_tW_v,\qquad
 k_t=\operatorname{softmax}(x_tW_k)
 \]
 \[
 S_t^i=\begin{cases}
 \Lambda_t S_{t-1}^i+e_t^i k_t^\top v_t,&i\in\TT_t,\\
 S_{t-1}^i,&i\notin\TT_t,
 \end{cases}
 \qquad o_t=\sum_{i\in\TT_t}e_t^i q_tS_t^i
 \]
 \begin{itemize}
 \item $S_t^i\in\mathbb R^{c\times d}$; $q_t,k_t\in\mathbb R^{1\times c}$; $e_t\in\mathbb R^N$.
 \item Gate values carry gradients through writing and reading.
 \item Within-partition softmax is soft allocation, not another hard top-$k$.
 \end{itemize}
 \papersource{Sec. 4.1, Eqs. (9)--(14), PDF pp. 7--8. Shared-partition contribution omitted here.}
\end{frame}

\begin{frame}{A memory decays only when its partition is visited}
 \begin{block}{A local update clock}
 Dense recurrence: one decay per token.\par
 Sparse recurrence: one decay per selected-partition update.
 \end{block}
 \begin{center}
 \begin{tabular}{lcc}
 \toprule
 Teaching example & Decay events & Retained multiplier \\
 \midrule
 Dense, 100 intervening tokens & 100 & $0.99^{100}\approx0.366$ \\
 Partition selected 10 times & 10 & $0.99^{10}\approx0.904$ \\
 \bottomrule
 \end{tabular}
 \end{center}
 Reading still requires routing back to the relevant partition.\par
 Long idle retention does not protect against later interfering writes.
 \papersource{Derived illustration of Eq. (13); Appendix A, Proposition 4, PDF pp. 24--25. Not experimental data.}
\end{frame}

\begin{frame}{State storage and active computation scale differently}
 \begin{columns}[T]
 \column{0.48\textwidth}
 \begin{block}{Store more}
 $N$ routed states + 1 shared state\par
 $\sim(N+1)cd$ elements / head
 \end{block}
 \begin{block}{Touch fewer}
 $k$ selected states + shared state\par
 $\sim(k+1)cd$ state work / step
 \end{block}
 \column{0.49\textwidth}
 \begin{itemize}
 \item \texttt{n4k1}: four routed partitions, select one; shared partition stays active.
 \item Shared QKV; rank-64 Q/K adaptation for the shared partition.
 \item Partition balance loss, coefficient $0.01$.
 \item Router parameters and routing work still grow with $N$.
 \end{itemize}
 \end{columns}
 \papersource{Secs. 4.1, 5.3; Appendix C, PDF pp. 8, 14, 27. Cost expressions are structural counts.}
\end{frame}

\begin{frame}{Varlen groups tokens while preserving causal order}
 \begin{columns}[T]
 \column{0.48\textwidth}
 \begin{block}{Short contexts: masking}
 Replicate activations across partitions.\par
 Mask inactive writes and reads.\par
 Parallelize over batch/head axes.
 \end{block}
 Avoids many tiny subsequences that underfill chunk kernels.
 \column{0.49\textwidth}
 \begin{block}{Long contexts: varlen}
 Group by sample and partition.\par
 Preserve time order within each group.\par
 Build new sequence offsets.
 \end{block}
 About $kL$ routed token records; one batched operator call, then scatter and sum.
 \end{columns}
 \vspace{3mm}
 \alert{Masking must also freeze inactive decay.}\par
 Prefill uses chunks; decoding updates only selected recurrent states.
 \papersource{Sec. 4.2; Appendix B, Algorithms 1--2, PDF pp. 8--9, 26--27.}
\end{frame}

\begin{frame}{At fixed $k$, varlen scales favorably with partition count}
 \centering\myfig{fig-06-ops_speed}{0.95}{0.68}
 \par\smallskip
 {\small Short sequences favor masking; long sequences favor varlen.\par
 A100, 8 heads, dimension 128, $K=1$, $N\in\{4,8,16\}$.}
 \papersource{Fig. 6, PDF p. 9. Forward + backward operator time; not end-to-end Transformer speedup.}
\end{frame}

\begin{frame}{Each training budget answers a different question}
 \begin{center}\small
 \begin{tabular}{p{0.26\linewidth}p{0.32\linewidth}p{0.32\linewidth}}
 \toprule
 Stage & Budget & Question \\
 \midrule
 Basic pretraining & 600M / 15B tokens;\newline 2B / 100B tokens, 8K & Recall and language quality \\
 Scale + extend & 2B: 2T + 250B tokens,\newline extend to 32K & Sufficient-data behavior \\
 Reasoning training & 80K distilled examples;\newline 5 epochs; GRPO 230 steps & Reasoning capability \\
 Transformer conversion & 2B, about 100B tokens\newline during 128K training & Preserve pretrained capability \\
 \bottomrule
 \end{tabular}\end{center}
 \small MHA--SwiGLU, no convolution. SSE-H: 15 SSE + 3 softmax layers at 2B.
 \papersource{Sec. 5, PDF pp. 10--12. State-scaling ablations separately use 2B / 20B tokens.}
\end{frame}

\begin{frame}{Pure SSE improves recall; hybrid SSE closes more of the gap}
 \begin{center}
 \begin{tabular}{lrrr}
 \toprule
 2B / 100B tokens & Wiki.$\downarrow$ & Commonsense$\uparrow$ & Recall$\uparrow$ \\
 \midrule
 Transformer & 16.46 & 53.55 & 73.00 \\
 GLA & 21.30 & 49.13 & 49.29 \\
 GDN & 17.08 & 54.05 & 53.79 \\
 \textbf{SSE-n4k1} & \textbf{16.92} & \textbf{54.57} & \textbf{61.46} \\
 GLA-H & 17.27 & 53.15 & 67.52 \\
 \textbf{SSE-H-n4k1} & \textbf{16.47} & \textbf{54.48} & \textbf{70.87} \\
 \bottomrule
 \end{tabular}
 \end{center}
 SSE gains \alert{7.67 percentage points} in recall over GDN.\par
 The Transformer recall gap is 11.54 points for SSE, 2.13 for SSE-H.
 \papersource{Table 1, PDF p. 10. Recall: mean of FDA, SWDE, SQuAD. Commonsense: six-task mean.}
\end{frame}

\begin{frame}{Harder 8K retrieval tasks reveal the memory advantage}
 \begin{center}
 \begin{tabular}{lrrr}
 \toprule
 2B / 100B tokens & S-NIAH-1 & S-NIAH-2 & S-NIAH-3 \\
 \midrule
 Transformer & 100.0 & 100.0 & 100.0 \\
 GLA & 87.6 & 23.2 & 16.2 \\
 GDN & 100.0 & 8.2 & 9.0 \\
 \textbf{SSE-n4k1} & \textbf{100.0} & \textbf{85.2} & \textbf{62.2} \\
 GLA-H & 100.0 & 100.0 & 70.8 \\
 \textbf{SSE-H-n4k1} & \textbf{100.0} & \textbf{100.0} & \textbf{97.4} \\
 \bottomrule
 \end{tabular}
 \end{center}
 Pure SSE still loses accuracy as context grows.\par
 Sparse state expansion and full-attention layers provide complementary gains.
 \papersource{Table 2, PDF p. 11. Zero-shot RULER, 8K columns; all scores in percent.}
\end{frame}

\begin{frame}{After 2T-token training, task-level gains remain mixed}
 \begin{columns}[T]
 \column{0.48\textwidth}
 \begin{block}{Broad evaluation after 32K extension}
 Ten-task average: \alert{45.6 vs. 45.1}\par
 SSE-H vs. Transformer
 \end{block}
 \begin{itemize}
 \item MMLU: 54.5 vs. 52.6
 \item C-Eval: 59.7 vs. 55.9
 \item GSM8K: 49.2 vs. 50.6
 \end{itemize}
 \column{0.49\textwidth}
 \begin{block}{Selected RULER tasks at 32K}
 S-NIAH-3: \alert{90.2 vs. 67.4}\par
 MQ-NIAH: \alert{44.6 vs. 28.6}\par
 MV-NIAH: \alert{40.6 vs. 18.1}
 \end{block}
 S-NIAH-1 is lower: 93.6 vs. 99.4.\par
 Multi-query/value recall remains difficult in absolute terms.
 \end{columns}
 \papersource{Table 3, PDF p. 11; Table 11, PDF p. 28. 2T pretraining + 250B extension tokens.}
\end{frame}

\begin{frame}{Conversion preserves more retrieval than GLA-H}
 \begin{center}
 \begin{tabular}{lrrrr}
 \toprule
 2B model & Quest & Qampari & Table Query & Average \\
 \midrule
 Transformer & 49.1 & 71.5 & 60.6 & 60.4 \\
 Converted GLA-H & 25.6 & 56.6 & 48.2 & 43.5 \\
 Converted SSE-H & 35.6 & 70.6 & 50.0 & 52.1 \\
 \bottomrule
 \end{tabular}
 \end{center}
 \begin{itemize}
 \item Conversion during 128K training, using about 100B tokens.
 \item Evaluation at 32K: +8.6 points over GLA-H, $-$8.3 vs. Transformer.
 \item Qampari is largely preserved; Quest still loses 13.5 points.
 \end{itemize}
 \papersource{Sec. 5.1, Table 4, PDF pp. 11--12. SSE-H uses n4k1.}
\end{frame}

\begin{frame}{Matched training supports comparable mathematical reasoning}
 \begin{center}\small
 \begin{tabular}{lrrrrr}
 \toprule
 Model & AIME24 & AIME25 & MATH500 & Olympiad & AMC23 \\
 \midrule
 Qwen3-1.7B Thinking & 48.3 & 36.8 & 93.4 & -- & -- \\
 R1-Distill-Qwen-7B & 55.5 & 39.2 & 92.8 & -- & -- \\
 \midrule
 Authors' Transformer-2B & 64.1 & 52.8 & 93.0 & 83.3 & 92.0 \\
 \textbf{Authors' SSE-H-2B} & \textbf{64.5} & \textbf{50.2} & \textbf{92.1} & \textbf{85.7} & \textbf{91.4} \\
 \bottomrule
 \end{tabular}\end{center}
 \begin{itemize}
 \item AIME gains over Qwen3: +16.2 / +13.4 points; training differs.
 \item Matched Transformer difference: +0.4 / $-$2.6 points.
 \item AIME uses \alert{avg@32}, not pass@32; temperature 1.0, top-p 0.7.
 \end{itemize}
 \papersource{Sec. 5.2, Table 5, PDF p. 12. ``Olympiad'' abbreviates OlympiadBench.}
\end{frame}

\begin{frame}{State scaling improves recall while active work also grows}
 \begin{columns}[c]
 \column{0.54\textwidth}
 \centering\myfig{fig-07-state_scale}{1}{0.66}
 \column{0.44\textwidth}
 \begin{itemize}
 \item 2B models, 20B tokens.
 \item More partitions improve recall at fixed $k/N$.
 \item Larger $k/N$ also improves recall in the plotted range.
 \item \alert{Fixed ratio means $k$ grows with $N$.} This is not the fixed-$K$ runtime comparison.
 \end{itemize}
 \end{columns}
 \papersource{Sec. 5.3, Fig. 7, PDF pp. 12--13. Original figure; limited configurations, not a fitted scaling law.}
\end{frame}

\begin{frame}{Softmax keys and shared parameters both improve recall}
 \begin{center}
 \begin{tabular}{lrr}
 \toprule
 600M / 15B tokens unless noted & Recall mean & Change \\
 \midrule
 SSE with SiLU keys & 24.23 & reference \\
 SSE with softmax keys & 31.16 & +6.93 \\
 \midrule
 SSE with shared parameters & 31.16 & reference \\
 Independent parameters, 890M total & 25.95 & $-$5.21 \\
 \bottomrule
 \end{tabular}
 \end{center}
 \begin{itemize}
 \item More independent parameters do not replace useful memory organization.
 \item Unshared non-embedding parameters rise from 300M to 580M.
 \item Commonsense changes only from 42.91 to 42.99 in that comparison.
 \end{itemize}
 \papersource{Tables 6 and 8, PDF pp. 13--14. All differences are percentage points.}
\end{frame}

\begin{frame}{The gate works best when used on both sides}
 \begin{center}
 \begin{tabular}{lrr}
 \toprule
 SSE gating ablation, 2B / 100B tokens & Recall mean & Commonsense \\
 \midrule
 \textbf{Write + read} & \textbf{56.63} & \textbf{53.63} \\
 Neither & 51.87 & 53.01 \\
 Write only & 50.92 & 52.82 \\
 Read only & 53.96 & 52.91 \\
 \bottomrule
 \end{tabular}
 \end{center}
 \begin{itemize}
 \item Joint write/read weighting beats either isolated use.
 \item The full row here is 56.63; Table 1 reports 61.46.
 \item v2 does not explain that difference: compare variants within this table.
 \end{itemize}
 \papersource{Table 7, PDF p. 14; Table 1, PDF p. 10.}
\end{frame}

\begin{frame}{State diagnostics support more selective memory use}
 \begin{columns}[c]
 \column{0.48\textwidth}
 \centering\myfig{fig-05-state_entropy}{1}{0.50}
 \par\small Energy spread over singular directions\par is higher on average for SSE.
 \column{0.48\textwidth}
 \centering\myfig{fig-08-row_sparsity}{1}{0.50}
 \par\small Key values below $10^{-5}$:\par routed 41.6\%, shared 4.9\%, GLA $\sim$0.
 \end{columns}
 \vspace{3mm}
 Near-zero key entries are not exact hard zeros.\par
 Representation diversity and retrieval accuracy are separate measurements.
 \papersource{Figs. 5 and 8, PDF pp. 8, 13. Original figures.}
\end{frame}

\begin{frame}{SSE makes state capacity an explicit design variable}
 \begin{enumerate}
 \item \textbf{Mechanism:} route information, pause irrelevant decay,\par
 and share parameters across larger recurrent memories.
 \item \textbf{Evidence:} stronger retrieval than linear baselines;\par
 hybrid models retain competitive post-trained reasoning.
 \item \textbf{Tradeoff:} larger stored states and routing work;\par
 hybrid layers still require context-growing KV caches.
 \end{enumerate}
 \vspace{3mm}
 \begin{block}{Questions for follow-up}
 Fixed-compute state scaling; state-aware routing;\par
 SSE combined with delta-rule transitions; checkpoint-based state expansion.
 \end{block}
 \papersource{Secs. 5--6, PDF pp. 10--14. SSE-on-GDN is future work in v2; GDN is already a baseline.}
\end{frame}

\begin{frame}{Paper and implementation sources}
 \begin{itemize}
 \item Yuqi Pan et al.\par
 \emph{Scaling Linear Attention with Sparse State Expansion}.\par
 \href{https://arxiv.org/abs/2507.16577v2}{arXiv:2507.16577v2}, revised 1 October 2025.
 \item \href{https://proceedings.iclr.cc/paper_files/paper/2026/hash/d3b8ce5e27b1c622d1b3da22b215e59b-Abstract-Conference.html}{ICLR 2026 official proceedings}:\par
 \emph{Scaling Linear Attention Capacity with Sparse State Expansion}.
 \item \href{https://proceedings.iclr.cc/paper_files/paper/2026/file/d3b8ce5e27b1c622d1b3da22b215e59b-Supplemental-Conference.zip}{Author implementation in the official supplement}:\par
 SSE-H modeling/configuration and masking kernel code.\par
 External project dependencies remain; not a standalone training package.
 \end{itemize}
 \vspace{2mm}
 Publication/code checked on 2 November 2026.\par
 Figures are author originals; tables select and retypeset cited results.
\end{frame}

\appendix

\begin{frame}{Backup: the classification argument is geometric}
 \[
 C^i=\{x:xW_k^i>k_{th}^i\},\qquad \|x\|^2=d
 \]
 \[
 \theta^i=\arccos\frac{k_{th}^i}{\sqrt d\|W_k^i\|}
 \quad\Longrightarrow\quad
 x_rx_s^\top>d\cos(2\theta^i),\quad x_r,x_s\in C^i.
 \]
 \begin{itemize}
 \item A high threshold restricts each class to a narrow cone.
 \item Between-class separation additionally needs separated classifier directions.
 \item Proposition 2 studies mixing under orthogonal $W_v$, nonnegative weights, and separated classes.
 \end{itemize}
 \papersource{Appendix A, Propositions 1--2, PDF pp. 21--23. These assumptions restrict the theoretical claim.}
\end{frame}

\begin{frame}{Backup: two theoretical claims need their conditions}
 \begin{block}{Output distinguishability: Proposition 3}
 The stated row-inner-product bound omits normalization present in the proof.\par
 If every state row is $(2,0)$, output cosine is 1 but row inner products are 4.\par
 The proof also assumes nonnegative queries; SSE uses a linear query.
 \end{block}
 \begin{block}{Unbounded retention: Proposition 4}
 With all-ones query/key, a channel whose decay stays exactly 1 can retain\par
 its contribution across an arbitrarily long idle interval. The threshold\par
 must not exceed that contribution; later writes can still interfere.
 \end{block}
 \papersource{Appendix A, PDF pp. 23--25. Counterexample is reader analysis of the v2 statement.}
\end{frame}

\begin{frame}{Backup: selected gate mass is usually below one}
 \[
 e_t=\operatorname{softmax}(x_tW_e),\qquad
 \sum_j k_t^j=1
 \]
 \[
 \boxed{\sum_{i\in\TT_t}\sum_j e_t^ik_t^j
       =\sum_{i\in\TT_t}e_t^i\leq1}
 \]
 \begin{itemize}
 \item Full-partition softmax followed by top-$k$ does not renormalize selected weights.
 \item v2 p. 8 describes total classification intensity as 1; Eqs. (9)--(14) generally give less than 1.
 \item The conference supplement retains original softmax gate values.
 \item Restoring a sum of 1 would change both scaling and gate gradients.
 \end{itemize}
 \papersource{Sec. 4.1, PDF pp. 7--8; official supplementary model code. Equation-based clarification.}
\end{frame}

\begin{frame}{Backup: row-sparse variants improve synthetic recall}
 \centering\myfig{fig-09-mqar}{0.82}{0.70}
 \par\small GLA, GSA, MetaLA; solid lines: top-$k$ variants.\par
 Setting 1 inserts zeros; setting 2 inserts random noise. Two layers, two heads, $k=d/4$.
 \papersource{Appendix A, Fig. 9, PDF p. 25. Original figure.}
\end{frame}

\begin{frame}{Backup: row similarity and effective receptive fields}
 \centering\myfig{fig-03-state_sim}{0.90}{0.35}
 \par\small Lower off-diagonal similarity supports more distinct state rows.
 \begin{columns}[c]
 \column{0.49\textwidth}
 \centering\myfig{fig-10-crf_comparison}{1}{0.35}
 \column{0.48\textwidth}
 \small Receptive-field probe: pretrained 2B models, 8K training, last 128 tokens, gating threshold 0.001.\par
 SSE has broader mean fields across layers; shaded bands are standard deviations.
 \end{columns}
 \papersource{Fig. 3, PDF p. 5; Appendix C, Fig. 10, PDF p. 28. Original figures.}
\end{frame}

\begin{frame}{Backup: all six RULER tasks at 32K}
 \begin{center}
 \begin{tabular}{lrr}
 \toprule
 Task & Transformer-2B & SSE-H-n4k1-2B \\
 \midrule
 S-NIAH-1 & 99.4 & 93.6 \\
 S-NIAH-2 & 70.8 & 84.4 \\
 S-NIAH-3 & 67.4 & 90.2 \\
 MK-NIAH-1 & 55.4 & 64.6 \\
 MQ-NIAH & 28.6 & 44.6 \\
 MV-NIAH & 18.1 & 40.6 \\
 \bottomrule
 \end{tabular}
 \end{center}
 SSE-H wins five of six tasks in this comparison.\par
 Both models remain far from perfect multi-query/value recall.
 \papersource{Appendix C, Table 11, PDF p. 28. Zero-shot; after long-context extension.}
\end{frame}

\begin{frame}{Backup: 12B conversion uses a different hybrid pattern}
 \begin{block}{Layer pattern}
 SSE $\rightarrow$ SWA $\rightarrow$ SSE $\rightarrow$ SWA $\rightarrow$ Softmax\par
 Convert during 128K training, then supervised distillation.
 \end{block}
 \begin{center}\small
 \begin{tabular}{lrrrrr}
 \toprule
 12B model & AIME24 & AIME25 & MATH500 & Olympiad & AMC23 \\
 \midrule
 Transformer & 75.7 & 58.7 & 95.8 & 82.7 & 97.2 \\
 SSE-H-n4k1 & 74.3 & 57.3 & 96.0 & 85.3 & 96.3 \\
 \bottomrule
 \end{tabular}\end{center}
 \vspace{2mm}
 Similar overall mathematical performance, with mixed task differences.\par
 SWA denotes sliding-window attention; this is not the 2B five-to-one pattern.
 \papersource{Appendix C, Table 10, PDF p. 27.}
\end{frame}

\end{document}