% Based on the arxiv-paper-reader Beamer template.
% Main talk: frames 1--21, approximately 25 minutes. Remaining frames: backup.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\spaceditems}{\setlength{\itemsep}{0.6em}}
\newcommand{\mat}[1]{\mathbf{#1}}

\title[Attention as contextual memory]{The Evolution of Attention\\in Large Language Models}
\subtitle{Mechanisms, Trade-offs, and Emerging Trends\\A memory-centric reading}
\author{Paper by Zhentao Tan, Jingyi Shen, Yanbo Li,\\Yao Liu, Yue Wu, and Jieping Ye}
\institute{Alibaba Token Hub, Alibaba Group\\\medskip Independent reading presentation}
\date{arXiv:2609.39661v1 $\mid$ Reading date: October 4, 2026}

\begin{document}

% 1
\begin{frame}[plain]
\begin{center}
{\usebeamerfont{title}\color{AcademicBlue}The Evolution of Attention\\in Large Language Models\par}
\medskip
{\large Mechanisms, Trade-offs, and Emerging Trends\par}
\medskip
Paper by Zhentao Tan, Jingyi Shen, Yanbo Li,\\
Yao Liu, Yue Wu, and Jieping Ye\\[2mm]
{\small Alibaba Token Hub, Alibaba Group\par}
\medskip
{\small Independent reading presentation $\mid$ October 4, 2026\\
\href{https://arxiv.org/abs/2609.39661v1}{arXiv:2609.39661v1}\par}
\bigskip
\alert{The design unit is the lifecycle of contextual memory.}
\end{center}
\end{frame}

% 2
\begin{frame}{Attention efficiency involves three different costs}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.31\textwidth}
\begin{block}{Prefill}
Dense causal interactions:\[
O(L^2d)
\]
Many query--key pairs.
\end{block}
\end{column}
\begin{column}{0.31\textwidth}
\begin{block}{Decoding}
Per-step attention work:\[
O(Ld)
\]
Growing KV traffic.
\end{block}
\end{column}
\begin{column}{0.31\textwidth}
\begin{block}{Retrieval quality}
Longer context expands the candidate set.\\[2mm]
Relevant evidence can be harder to recover.
\end{block}
\end{column}
\end{columns}
\medskip
\textbf{Survey question:} how do architectures organize memory to balance these costs and capabilities?
\papersource{Paper, Secs. 1 and 4, PDF pp. 4 and 15. $L$: sequence length; $d$: attention width.}
\end{frame}

% 3
\begin{frame}{Five functions make architectural changes comparable}
\renewcommand{\arraystretch}{1.35}
\begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.26\linewidth}>{\raggedright\arraybackslash}p{0.68\linewidth}@{}}
\toprule
\textbf{Function} & \textbf{Question}\\
\midrule
Representation & What survives, and in what form?\\
Update & What is appended, retained, corrected, or merged?\\
Access & Which stored units may participate in this read?\\
Readout & How does the query extract their information?\\
Integration & How do completed readouts form the output?\\
\bottomrule
\end{tabular}\par
\vspace{3mm}
These are analytical roles. One operation may change several roles, and the implementation order can vary.
\papersource{Paper, Sec. 2, Eqs. (1)--(5), Fig. 1, PDF pp. 6--10.}
\end{frame}

% 4
\begin{frame}{History coverage and token addressability are different}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.48\textwidth}
\begin{block}{Explicit memory}
\[
\{(k_i,v_i)\}_{i\le t}
\]
The query can select individual entries.\\[2mm]
Head/channel sharing reduces the cost per token.\\[2mm]
Summaries and slots may lose original token identity.
\end{block}
\end{column}
\begin{column}{0.48\textwidth}
\begin{block}{Recurrent state}
\[
S_t=f(S_{t-1},x_t)
\]
The entire history may influence a bounded state.\\[2mm]
Original tokens are usually no longer separately selectable.\\[2mm]
Capacity and interference govern recovery.
\end{block}
\end{column}
\end{columns}
\papersource{Paper, Sec. 2.1, PDF p. 7; Secs. 3.2 and 5.}
\end{frame}

% 5
\begin{frame}{KV compression targets heads, channels, or depth}
\[
M_{\mathrm{KV}}=2BTL_{\mathrm{KV}}H_{\mathrm{KV}}d_hs
\]
\begin{itemize}\spaceditems
\item \textbf{MQA/GQA:} reduce stored KV heads $H_{\mathrm{KV}}$.
\item \textbf{MLA:} cache a compact latent plus a positional component.
\item \textbf{CLA:} reduce independent KV-producing layers $L_{\mathrm{KV}}$.
\end{itemize}
\begin{block}{Illustrative KV-only calculation}
At 32K tokens, 32 layers, $d_h=128$, 2-byte scalars, $B=1$:\\[1mm]
32 KV heads: \textbf{16 GiB}\quad $\to$\quad 8 KV heads: \textbf{4 GiB}\\
With 4-layer KV sharing: \textbf{1 GiB}.
\end{block}
\papersource{Paper, Sec. 3.1, Eq. (7), PDF pp. 11--12. Example calculated for this presentation; $T=32768$. All three retain linear growth in $T$ when all tokens are cached.}
\end{frame}

% 6
\begin{frame}{Memory compression and output gating change different things}
\begin{itemize}\spaceditems
\item \textbf{Fixed-capacity memory:} RMT/FAM latent states or Trellis/Lattice KV slots overwrite or consolidate history.
\item \textbf{Dynamic resolution:} recent tokens stay precise; older context becomes summaries or compressed entries.
\item \textbf{Readout modulation:} change scores or attention maps, as in Differential Attention and FoX.
\item \textbf{Integration modulation:} gate or route completed head outputs, as in Gated Attention and MoH.
\end{itemize}
\medskip
\alert{A post-readout gate cannot remove KV loads and score computations that already occurred.}
\papersource{Paper, Secs. 3.2--3.3, PDF pp. 12--14. Actual savings depend on operations skipped by the implementation.}
\end{frame}

% 7
\begin{frame}{Sparse attention budgets access to represented memory}
\[
\mathcal I_t=\operatorname{Select}(e_{t,:},\mathcal U_t;b_t),\qquad
r_t=\operatorname{softmax}\!\left(\frac{q_tK_{\mathcal I_t}^{\top}}{\sqrt d}\right)V_{\mathcal I_t}
\]
\begin{itemize}\spaceditems
\item \textbf{Structure:} prescribe a mask or instantiate a profiled pattern.
\item \textbf{Self-routing:} reuse backbone geometry, pooled keys, or summaries.
\item \textbf{Auxiliary proxy:} learn a separate address space for selection.
\end{itemize}
\begin{block}{Cost accounting}
Evidence + index scan + Top-$k$ + KV gather + readout + refresh.
\end{block}
\papersource{Paper, Sec. 4, Eqs. (8)--(9), PDF pp. 15--26. $e$: selection scores; $\mathcal U$: available units; $b_t$: access budget. Cost list is a reading synthesis.}
\end{frame}

% 8
\begin{frame}{QSA shortens the index scan by pooling before scoring}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.31\textwidth}
\begin{block}{DSA}
Token index keys\\[1mm]
$\downarrow$\\
Token scores\\[1mm]
$\downarrow$\\
Weighted head fusion\\[1mm]
$\downarrow$\\
Shared token Top-$k$
\end{block}
\end{column}
\begin{column}{0.31\textwidth}
\begin{block}{MSA}
Token index keys\\[1mm]
$\downarrow$\\
Token scores\\[1mm]
$\downarrow$\\
Block max-pooling\\[1mm]
$\downarrow$\\
Block Top-$k$ per group
\end{block}
\end{column}
\begin{column}{0.31\textwidth}
\begin{block}{QSA}
Token index keys\\[1mm]
$\downarrow$\\
Average-pool keys\\[1mm]
$\downarrow$\\
Micro-block scores\\[1mm]
$\downarrow$\\
Shared block Top-$k$
\end{block}
\end{column}
\end{columns}
\medskip
DSA, MSA, and QSA retrieve original-token content.\\
\textbf{CSA additionally compresses the global content itself.}
\papersource{Redrawn schematically from Paper, Fig. 7 and Eqs. (29)--(32), PDF pp. 21--24. Local/current-block inclusion omitted from the schematic.}
\end{frame}

% 9
\begin{frame}{Content, addresses, and selected indices can refresh separately}
\renewcommand{\arraystretch}{1.35}
\begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.15\linewidth}>{\raggedright\arraybackslash}p{0.30\linewidth}>{\raggedright\arraybackslash}p{0.48\linewidth}@{}}
\toprule
\textbf{CSA2 mode} & \textbf{Global KV / index keys} & \textbf{Selected support}\\
\midrule
Full & Fresh & Full index scan and new Top-$k$\\
Reindex & Reuse from Full & New selection within shared candidate pool\\
Reuse & Reuse from Full & Inherit latest selection\\
\bottomrule
\end{tabular}\par
\vspace{3mm}
All modes still form their own main query and local-window KVs.
\begin{itemize}\spaceditems
\item \textbf{IndexCache:} reuse indices while layers retain their own content.
\item \textbf{Temporal reuse:} recall prior supports, then rerank for the new query.
\end{itemize}
\papersource{Paper, Sec. 4.4, Eqs. (33)--(37), PDF pp. 24--26. CSA2 candidate-pool reuse described for decoding.}
\end{frame}

% 10
\begin{frame}{Linear attention changes what is stored}
For a separable kernel $K(q,k)=\phi(q)^\top\phi(k)$:
\[
S_t=S_{t-1}+\phi(k_t)v_t^\top,\qquad
z_t=z_{t-1}+\phi(k_t)
\]
\[
o_t=\frac{\phi(q_t)^\top\widehat S_t}
{\phi(q_t)^\top\widehat z_t+\varepsilon}
\]
\begin{itemize}\spaceditems
\item $S_t\in\mathbb R^{d_k\times d_v}$ compresses key--value associations.
\item Fixed dimensions give context-independent persistent state.
\item Parallel/chunkwise training can compute the recurrence efficiently.
\end{itemize}
\papersource{Paper, Eqs. (38)--(39), PDF p. 27. Hats denote the visible read state; normalization requires an appropriate kernel. Later variants may use unnormalized readout.}
\end{frame}

% 11
\begin{frame}{Delta updates correct the association already stored}
\begin{block}{Residual-dependent write}
\[
S_t=S_{t-1}+\beta_tk_t\bigl(v_t^\top-k_t^\top S_{t-1}\bigr)
\]
\end{block}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.48\textwidth}
\textbf{Additive update}\\[2mm]
Writes the current value regardless of the old prediction.\\[2mm]
Repeated associations can accumulate and interfere.
\end{column}
\begin{column}{0.48\textwidth}
\textbf{Delta update}\\[2mm]
Fits the current key--value pair.\\[2mm]
GDN adds scalar decay; KDA adds channel-wise decay before correction.
\end{column}
\end{columns}
\medskip
With $\|k\|=1$ and $\beta=1$, values $1\to3$ at the same key give:\\
additive readout $4$; Delta readout $3$ \textit{(teaching example)}.
\papersource{Paper, Sec. 5.1, Tables 7--8, PDF pp. 27--32. Better updates still operate within finite capacity.}
\end{frame}

% 12
\begin{frame}{More memory can mean more rows or more time intervals}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.48\textwidth}
\begin{block}{Capacity expansion}
\textbf{SSE:} grouped state rows with sparse group activation.\\[2mm]
\textbf{SDM:} flat rows with separate read and write routers.\\[2mm]
Total stored capacity and active work are different budgets.
\end{block}
\end{column}
\begin{column}{0.48\textwidth}
\begin{block}{Temporal expansion}
\textbf{Log-Linear:} $O(\log t)$ multiscale states.\\[2mm]
\textbf{DLA:} content-based segments, $M_t\le K$.\\[2mm]
\textbf{MHLA:} fixed-block summaries; count may grow.
\end{block}
\end{column}
\end{columns}
\medskip
Routing must align what gets written with what future queries can recover.
\papersource{Paper, Secs. 5.2--5.3, Tables 9--10, Fig. 8, PDF pp. 32--35.}
\end{frame}

% 13
\begin{frame}{Selective SSMs make state dynamics depend on the input}
\begin{block}{Time-invariant kernel}
\[
h_t=\bar Ah_{t-1}+\bar Bx_t,\qquad r_t=Ch_t
\]
From $h_0=0$: $r_t=\sum_{i\le t}C\bar A^{t-i}\bar Bx_i$.
\end{block}
\begin{itemize}\spaceditems
\item S4/S4D structure the transition; S5 uses a joint MIMO system.
\item Mamba generates $\Delta_t,B_t,C_t$ from the current input; $\bar A_t=\exp(\Delta_t A)$.
\item Content-dependent controls require selective sequence computation; generator weights stay fixed at inference.
\end{itemize}
\papersource{Paper, Sec. 6.1, Eqs. (55)--(56), PDF pp. 37--38. $h_t$: recurrent state; $x_t$: input; $r_t$: readout.}
\end{frame}

% 14
\begin{frame}{SSD gives recurrent and matrix views of the same kernel}
\[
H_t=a_tH_{t-1}+b_tu_t^\top,\qquad r_t=H_t^\top c_t
\]
\centering
\includegraphics[width=0.83\linewidth,height=0.36\textheight,keepaspectratio]{figures/fig-11.pdf}
\[
r_t=\sum_{i\le t}\underbrace{(c_t^\top b_i)}_{\text{read--write match}}
\underbrace{\left(\prod_{j=i+1}^{t}a_j\right)}_{\text{intervening decay}}u_i
\]
\papersource{Paper, Fig. 11, Eqs. (57)--(58), PDF pp. 38--39. $H_t\in\mathbb R^{N\times P}$; $b_t,c_t\in\mathbb R^N$; $u_t\in\mathbb R^P$; $a_t$ is scalar per head.}
\end{frame}

% 15
\begin{frame}{Hybrid architecture chooses where capabilities meet}
\renewcommand{\arraystretch}{1.38}
\begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.17\linewidth}>{\raggedright\arraybackslash}p{0.38\linewidth}>{\raggedright\arraybackslash}p{0.38\linewidth}@{}}
\toprule
\textbf{Granularity} & \textbf{Decision} & \textbf{Example}\\
\midrule
Layer & Place different mixers along depth & Jamba, Samba; conversion/search\\
Head & Allocate head/channel capacity & Hymba, DuoAttention, HydraHead\\
Branch & Fuse identifiable memory paths & Infini-attention, DART, Titans-MAG\\
Token & Choose storage or operation per token & LoLCATs, LoLA, NAtS-L\\
\bottomrule
\end{tabular}\par
\vspace{3mm}
The granularities can coexist. A hybrid is a composition of mechanisms or access regimes, rather than another memory operator.
\papersource{Paper, Sec. 7, Fig. 13 and Tables 13--16, PDF pp. 42--48.}
\end{frame}

% 16
\begin{frame}{Hybrid records rise within the curated architecture inventory}
\begin{center}
\renewcommand{\arraystretch}{1.4}
\begin{tabular}{lrrrr}
\toprule
Period & Records & Single-family & Hybrid & Cross-layer\\
\midrule
2022--2023 & 6 & 6 & 0 & 0\\
2024--2025 & 25 & 16 & 9 (36\%) & 0\\
2026 to Sep 22 & 27 & 10 & 17 (63\%) & 5 (19\%)\\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}\spaceditems
\item 59 release records, 14 lineages; 58 records are classifiable.
\item Kimi k1.5 is excluded from percentages because its attention is undisclosed.
\item Cross-layer reuse overlaps with the other labels: \textbf{do not add the columns}.
\end{itemize}
\papersource{Paper, Table 17, PDF p. 50; Table S1, pp. 57--58. Curated release counts, not market share or controlled performance measurements.}
\end{frame}

% 17
\begin{frame}{The frozen frontier sample retains diverse explicit paths}
\renewcommand{\arraystretch}{1.35}
\begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.38\linewidth}>{\raggedright\arraybackslash}p{0.57\linewidth}@{}}
\toprule
\textbf{Selected endpoint} & \textbf{Documented attention structure}\\
\midrule
K2-Horizon-375B-A23B & Full GQA\\
GLM-5.3 (max) & MLA-based DSA + index sharing\\
Qwen3.8-Flash-Next & 3 GDN : 1 QSA\\
Kimi K3 (max) & 3 KDA : 1 Gated MLA\\
DeepSeek-V4.1-Flash (max) & CSA2 + local window; cross-layer reuse\\
\bottomrule
\end{tabular}\par
\vspace{3mm}
All 11 selected architectures retain an explicit token-retrieval path.\\
That path may be local; remote content may already be compressed.
\papersource{Selected rows from Paper, Table 18, PDF p. 50. Artificial Analysis v4.3.2 snapshot: Sep 22, 2026. Describes coexistence; does not isolate causal effects of attention design.}
\end{frame}

% 18
\begin{frame}{Network depth governs both placement and persistence}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.48\textwidth}
\begin{block}{Capability placement}
Which layer performs explicit retrieval, sparse access, or recurrent propagation?\\[3mm]
Layer-wise hybrids coordinate through the hidden-state stream.
\end{block}
\end{column}
\begin{column}{0.48\textwidth}
\begin{block}{Artifact persistence}
Which KVs, index keys, or selected supports survive into later layers?\\[3mm]
Reuse schedules determine when each object is refreshed.
\end{block}
\end{column}
\end{columns}
\medskip
The inventory's five cross-layer records represent three base patterns across GLM, DeepSeek, and LongCat.
\papersource{Paper, Secs. 8.2 and 9.2, PDF pp. 49--50 and 52--53. Cross-layer reuse is a separate, overlapping architectural attribute.}
\end{frame}

% 19
\begin{frame}{Hypothesis: route persistent memory across four coordinates}
\[
\mathcal A_t=\mathcal T_t\times\mathcal L\times\mathcal S\times\mathcal G,
\qquad a=(\tau,\ell,s,g)
\]
\begin{center}
\renewcommand{\arraystretch}{1.25}
\begin{tabular}{ll}
Temporal scope $\tau$ & token, segment, long interval\\
Depth $\ell$ & producer or maintenance stage\\
Substrate $s$ & explicit KV, associative state, SSM state\\
Granularity $g$ & exact entries or coarse summaries\\
\end{tabular}
\end{center}
\medskip
\textbf{Sparse Write:} decide where information is retained.\\[1mm]
\textbf{Sparse Read:} decide which stored information serves a query.
\papersource{Author hypothesis, Sec. 9.3, Eqs. (68)--(70), PDF pp. 53--54. A logical address space, not a required Cartesian tensor or an evaluated new model.}
\end{frame}

% 20
\begin{frame}{Four proposed steps make the hypothesis testable}
\begin{enumerate}\spaceditems
\item \textbf{Choose representations by workload:} exact KV for precision-sensitive regions; summaries and indexes elsewhere.
\item \textbf{Separate construction from retrieval:} lightweight memory construction, then coarse-to-fine query-time access.
\item \textbf{Control lifecycle incrementally:} reuse, refresh, transform, or invalidate affected artifacts.
\item \textbf{Route memory and compute together:} start with a finite menu of memory, depth, and write-back policies.
\end{enumerate}
\medskip
Write decisions have delayed consequences: future queries can only retrieve what the memory system preserved.
\papersource{Author proposals, Sec. 9.3.4, PDF pp. 54--55. The complete multidimensional system remains unvalidated.}
\end{frame}

% 21
\begin{frame}{The survey is a design map for quality--cost trade-offs}
\begin{block}{What the paper contributes}
A shared vocabulary, a mechanism taxonomy, an architecture inventory, and a testable research hypothesis.
\end{block}
\begin{itemize}\spaceditems
\item Explicit memory preserves selectable units; recurrent memory compresses history into state.
\item Hybrid placement and artifact reuse make depth part of memory management.
\item Compare retrieval quality, persistent memory, prefill, decoding, and routing under matched budgets.
\end{itemize}
\medskip
\alert{Adoption counts do not establish an attention mechanism's performance advantage.}
\papersource{Paper, Secs. 8--10, PDF pp. 48--56. Main talk ends here; following slides are technical backup.}
\end{frame}

\appendix

% 22
\begin{frame}{Backup: routing scores can also control readout}
HiLS approximates the attention mass of chunk $u$:
\[
Z_{t,u}=\sum_{j\in\mathcal B_u}e^{s_{t,j}},\qquad
\widehat s_{t,u}=\frac{q_t(k'_u)^\top}{\sqrt d}+H(\alpha_u)
\]
\[
k'_u=\sum_j\alpha_{u,j}k_j,\quad
\widehat Z_{t,u}=e^{\widehat s_{t,u}},\quad
w_{t,j}=\frac{e^{s_{t,j}}}{Z_{t,c(j)}}
\frac{\widehat Z_{t,c(j)}}{Z_{t,\mathrm{loc}}+\sum_{u\in\mathcal S_t}\widehat Z_{t,u}}
\]
\begin{itemize}\spaceditems
\item Exact within-chunk weights; surrogate masses across selected chunks.
\item The surrogate participates in continuous forward weights, allowing language-model loss to train the summaries.
\end{itemize}
\papersource{Paper, Eqs. (23)--(28), PDF p. 20. $\alpha_u$: landmark-induced distribution; $H$: entropy; $c(j)$: chunk of token $j$; $\mathcal S_t$: selected distant chunks.}
\end{frame}

% 23
\begin{frame}{Backup: richer updates decouple retention, erasure, and writing}
Let $S\in\mathbb R^{d_k\times d_v}$ and $D_t=\operatorname{Diag}(\alpha_t)$.
\begin{align*}
\text{KDA:}\quad S_t
&=(I-\beta_tk_tk_t^\top)D_tS_{t-1}+\beta_tk_tv_t^\top\\[2mm]
\text{GDN2:}\quad S_t
&=[I-k_t(b_t\odot k_t)^\top]D_tS_{t-1}
+k_t(w_t\odot v_t)^\top\\[2mm]
\text{EDA:}\quad S_t
&=(I-\beta_tk_tk_t^\top)(I-\gamma_te_te_t^\top)D_tS_{t-1}
+\beta_tk_tv_t^\top
\end{align*}
\begin{itemize}
\item GDN2: separate key-side erase control $b_t$ and value-side write control $w_t$.
\item EDA: erase at $e_t$, then correct at $k_t$; operation order matters.
\end{itemize}
\papersource{Paper, Table 7, PDF p. 31; Sec. 5.1. Feature generation, normalization, and output gates omitted as in the survey's comparison.}
\end{frame}

% 24
\begin{frame}{Backup: Delta correction is one state-optimization step}
\[
\mathcal L_t(S)=\frac{\beta_t}{2}\|S^\top k_t-v_t\|_2^2
\]
\[
\nabla_S\mathcal L_t=\beta_tk_t(k_t^\top S-v_t^\top)
\]
\[
S-\nabla_S\mathcal L_t=(I-\beta_tk_tk_t^\top)S+\beta_tk_tv_t^\top
\]
\begin{itemize}\spaceditems
\item GDN/KDA optimize this target at a decayed state.
\item DeltaProduct applies multiple sequential corrections per token.
\item EDA first fits a zero target at an erase address, then fits the value.
\end{itemize}
\papersource{Paper, Eq. (51) and Table 8, PDF pp. 30--32. Equivalent objectives for the internal state; pretrained model weights are not the optimization variables here.}
\end{frame}

% 25
\begin{frame}{Backup: SSM state capacity and interface width are separate}
\[
H_t\in\mathbb R^{N\times P},\quad
C_t^{\mathrm{multi}}\in\mathbb R^{N\times R},\quad
Y_t=H_t^\top C_t^{\mathrm{multi}}\in\mathbb R^{P\times R}
\]
\begin{itemize}\spaceditems
\item Larger $N,P$ increase retained state; more directions $R$ enrich readout from the same principal state.
\item Direct multi-direction readout costs $O(NPR)$ arithmetic.
\item MIMOMamba uses a different matrix-valued algebraic construction.
\item HADES organizes temporal filters; MuonSSM conditions input-injection geometry.
\end{itemize}
\papersource{Paper, Sec. 6.2, Eq. (59), Fig. 12, PDF pp. 40--41. More read directions cannot recover information no longer encoded in the state.}
\end{frame}

% 26
\begin{frame}{Backup: hybrid allocation policies use different evidence}
\renewcommand{\arraystretch}{1.32}
\begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.34\linewidth}>{\raggedright\arraybackslash}p{0.60\linewidth}@{}}
\toprule
\textbf{Layer allocation} & \textbf{Selection signal}\\
\midrule
Predefined & Architecture designer specifies a schedule\\
Direct diagnosis & Original pretrained layer behavior\\
Alignment-based & Replacement behavior after representation alignment\\
Iterative distillation & KL reduction when restoring a layer in an adapted student\\
\bottomrule
\end{tabular}\par
\vspace{3mm}
\textbf{Head policies also differ:} offline function selection, input-adaptive mode selection, and depth-dependent head budgets.
\papersource{Paper, Secs. 7.1--7.2, Tables 13--14, PDF pp. 42--45.}
\end{frame}

% 27
\begin{frame}{Backup: branch fusion depends on the interface}
\begin{block}{Readout-level fusion: Infini-attention}
\[
A=\sigma(\beta)\odot A_{\mathrm{mem}}
+[1-\sigma(\beta)]\odot A_{\mathrm{dot}}
\]
Combine compressive-memory and local-attention contexts, then apply a shared multi-head output transformation.
\end{block}
\begin{block}{Output-level fusion: Titans-MAG}
\[
o_t^{\mathrm{MAG}}=\operatorname{Gate}(o_t^{\mathrm{SWA}},o_t^{\mathrm{mem}})
\]
Each branch first forms its own output; fusion defines the hybrid output.
\end{block}
\papersource{Paper, Sec. 7.3, Eqs. (61)--(65), Table 15, PDF p. 46. $A_{\mathrm{mem}}$: compressed-memory readout; $A_{\mathrm{dot}}$: local Softmax readout.}
\end{frame}

% 28
\begin{frame}{Backup: token routing determines memory fidelity over time}
\begin{itemize}\spaceditems
\item \textbf{Temporal boundary:} LoLCATs moves expired local tokens into a linear state; NHA uses long-term slots.
\item \textbf{Content score:} LoLA uses self-recall error; STILL uses self-saliency to retain selected historical KVs.
\item \textbf{Learned operation:} NAtS-L chooses Softmax or GDN at token/chunk granularity.
\end{itemize}
\[
z_i\in\{\mathrm{KV}_{\mathrm{recent}},\mathrm{KV}_{\mathrm{salient}},S_{\mathrm{compressed}}\}
\]
\medskip
The storage decision is made before all future queries are known.
\papersource{Paper, Sec. 7.4, Table 16 and Eqs. (66)--(67), PDF p. 47. $z_i$: architecture-level allocation decision.}
\end{frame}

% 29
\begin{frame}{Backup: frozen endpoint inventory (1/2)}
\renewcommand{\arraystretch}{1.38}
\begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.38\linewidth}>{\raggedright\arraybackslash}p{0.57\linewidth}@{}}
\toprule
\textbf{Endpoint} & \textbf{Attention structure}\\
\midrule
MiMo-V2.6-Pro & 60 SWA-GQA + 10 global-GQA layers\\
DeepSeek-V4.1-Flash (max) & CSA2 + SWA; cross-layer KV/index/support reuse\\
K2-Horizon-375B-A23B & Full GQA\\
GLM-5.3-Flash & 34 KDA + 11 compressed-indexer DSA layers\\
GLM-5.3 (max) & MLA-based DSA; IndexShare/IndexCache\\
Qwen3.8-Flash-Next & 3 GDN : 1 QSA\\
\bottomrule
\end{tabular}\par
\vspace{2mm}
\papersource{Paper, Table 18, PDF p. 50. Artificial Analysis Intelligence Index v4.3.2 snapshot captured Sep 22, 2026. Rows describe endpoint architectures, not a mechanism ranking.}
\end{frame}

% 30
\begin{frame}{Backup: frozen endpoint inventory (2/2)}
\renewcommand{\arraystretch}{1.42}
\begin{tabular}{@{}>{\raggedright\arraybackslash}p{0.38\linewidth}>{\raggedright\arraybackslash}p{0.57\linewidth}@{}}
\toprule
\textbf{Endpoint} & \textbf{Attention structure}\\
\midrule
DeepSeek-V4 Pro 0813 (max) & CSA/HCA layer-wise hybrid\\
Qwen3.8-2.4T-A95B & 3 GDN : 1 gated full-GQA layer\\
Qwen3.8-27B (xhigh) & 3 GDN : 1 gated full-GQA layer\\
Kimi K3 (max) & 3 KDA : 1 Gated MLA\\
MiniMax-M3 & 3 dense-attention + 57 MSA layers\\
\bottomrule
\end{tabular}\par
\vspace{3mm}
Reasoning settings describe endpoints; they are not separate architectures. The paper does not report index scores in this table.
\papersource{Paper, Table 18 and Sec. 8.3, PDF p. 50. Snapshot date: Sep 22, 2026.}
\end{frame}

% 31
\begin{frame}{Backup: source and reading route}
\textbf{Primary source}\\[2mm]
Zhentao Tan, Jingyi Shen, Yanbo Li, Yao Liu, Yue Wu, Jieping Ye.\\[1mm]
\textit{The Evolution of Attention in Large Language Models: Mechanisms, Trade-offs, and Emerging Trends.}\\[1mm]
\href{https://arxiv.org/abs/2609.39661v1}{arXiv:2609.39661v1}, September 30, 2026.
\medskip
\begin{itemize}
\item Framework: Sec. 2; explicit/sparse memory: Secs. 3--4.
\item State updates and SSMs: Secs. 5--6; composition: Sec. 7.
\item Architecture evidence: Sec. 8 and Appendix A.
\item Synthesis and proposals: Sec. 9; conclusion: Sec. 10.
\end{itemize}
\papersource{All mechanism and architecture claims in this presentation are attributed to the survey. Its cited primary literature is listed in the paper, PDF pp. 59--67.}
\end{frame}

\end{document}