% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}


\newcommand{\att}{\operatorname{Atten}}
\newcommand{\sm}{\operatorname{softmax}}
\title[MiTA]{Mixture-of-Top-$k$ Attention}
\subtitle{Efficient Attention via Scalable Fast Weights}
\author{Paper by Qishuai Wen, Zhiyuan Huang, Wei He,\\Xianghan Meng, and Chun-Guang Li}
\date{arXiv:2602.01219v5 (9 May 2026)\\Research discussion --- 21 September 2026}
\begin{document}
\begin{frame}\titlepage\end{frame}
\begin{frame}{Build a few sparse memories; reuse them across queries}

\begin{block}{Central idea}
A small set of landmark queries builds reusable top-$k$ key--value sets, plus a compressed global memory.
\end{block}
\begin{itemize}
\item Each query reads the shared memory and one routed set.
\item LRA: 58.19\% $\to$ 58.91\% mean accuracy; 10.6 $\to$ 2.4 training hours.
\item Evidence supports an efficiency--accuracy tradeoff, with task-dependent losses.
\end{itemize}

\papersource{Secs. 3.2 and 4.3; PDF pp. 5--6, 9; Table 5.}
\end{frame}
\begin{frame}{The missing middle: flexible but reusable sparse patterns}

\begin{center}
\begin{tabular}{lll}\toprule
Design & Pattern construction & Main constraint\\\midrule
Block routing & Contiguous K/V blocks & Rigid membership\\
Per-query top-$k$ & A private set per query & $N$ sets to construct\\
Compression & A small global memory & Fine details lost\\
MiTA & $m$ reusable top-$k$ sets & Shared pattern budget\\\bottomrule
\end{tabular}
\end{center}
\medskip
Here an \alert{expert is an input-dependent K/V set}, not an added FFN with persistent parameters.

\papersource{Secs. 1--2; PDF pp. 1--4; Table 1.}
\end{frame}
\begin{frame}{Attention can be viewed as an input-dependent wide MLP}

\[
\att(q,K,V)=V\sm(K^\top q/\sqrt d)
             =\frac{V\exp(\hat K^\top q)}{Z(q)}
\]
\[
\hat K=K/\sqrt d,\qquad Z(q)=\sum_{j=1}^{N}\exp(\hat k_j^\top q).
\]
\begin{itemize}
\item $K,V\in\mathbb R^{d\times N}$: width grows with sequence length $N$.
\item $N$ queries reading $N$ entries give quadratic arithmetic.
\item Caveat: the paper's $\hat V=V/Z(q)$ depends on the query. This is an algebraic view, not an ordinary fixed-weight MLP.
\end{itemize}

\papersource{Sec. 3.1; PDF pp. 4--5; Eqs. (1)--(4). Caveat: reading analysis.}
\end{frame}
\begin{frame}{Five design axes separate compression from routing}

\begin{enumerate}
\item \textbf{Strategy}: compress, route, or combine both.
\item \textbf{Count}: one shared expert, $m$ experts, or $N$ private experts.
\item \textbf{Type}: linear layer, attention/MLP, or a more general module.
\item \textbf{Construction}: locality, learned projection, landmark probing, etc.
\item \textbf{Topology}: how queries are assigned to experts.
\end{enumerate}
\medskip
MiTA: $m$ routed experts \alert{plus one shared expert of width $m$}.

\papersource{Table 1, Sec. 3.1, Appendix B; PDF pp. 2, 5, 15.}
\end{frame}
\begin{frame}{MiTA combines a global summary with original entries}

\begin{columns}[c]
\column{0.49\textwidth}
\includegraphics[width=\linewidth,height=0.67\textheight,keepaspectratio]{figures/fig02-method.pdf}
\column{0.47\textwidth}
\begin{enumerate}
\item Pool $Q$ into $m$ landmarks.
\item Probe all keys once per landmark.
\item Keep top-$k$ original K/V and form a weighted summary.
\item Route queries and jointly normalize the two memories.
\end{enumerate}
\end{columns}

\papersource{Original Figure 2, PDF p. 3; Sec. 3.2, pp. 5--6.}
\end{frame}
\begin{frame}{One landmark builds one deformable expert}

\[
\tilde Q=\operatorname{AvgPool}_{2D}(Q)\in\mathbb R^{d\times m},
\qquad S^{\mathrm{kv}}=K^\top\tilde Q/\sqrt d\in\mathbb R^{N\times m}
\]
\[
\mathcal I_i=\operatorname{Top}_k(K^\top\tilde q_i),\qquad
K^{(i)}=K_{:,\mathcal I_i},\quad V^{(i)}=V_{:,\mathcal I_i}.
\]
\begin{itemize}
\item Selection depends on both pooled queries and key content.
\item Membership can cross spatial boundaries and overlap across experts.
\item The retrieved K/V remain original entries, not pooled values.
\end{itemize}

\papersource{Sec. 3.2; PDF pp. 5--6; Eqs. (5)--(7).}
\end{frame}
\begin{frame}{The same landmarks also form the global memory}

\[
\tilde V=V\sm_N(S^{\mathrm{kv}})\in\mathbb R^{d\times m},\qquad
\tilde{\mathcal E}(q)=\att(q,\tilde Q,\tilde V).
\]
\begin{itemize}
\item Each landmark value summarizes all $N$ original values.
\item Landmark \emph{queries} now serve as compressed \emph{keys}.
\item $K^\top\tilde Q$ supports both top-$k$ selection and compression.
\item $\tilde Q^\top Q$ supports both routing and shared-memory lookup.
\end{itemize}
\medskip
Compression and routing share computations rather than acting as unrelated add-ons.

\papersource{Sec. 3.2; PDF p. 6; Eqs. (8)--(9).}
\end{frame}
\begin{frame}{A single softmax couples detail and global context}

\[
K^\star=[\tilde Q,K^{(e_1(q))},\ldots,K^{(e_s(q))}],\quad
V^\star=[\tilde V,V^{(e_1(q))},\ldots,V^{(e_s(q))}]
\]
\[
\operatorname{MiTA}(q)=V^\star\sm((K^\star)^\top q/\sqrt d).
\]
\begin{itemize}
\item Each query reads $m+ks$ entries; all experiments use $s=1$.
\item Sharing an expert does not force queries to share attention weights.
\item Separate branch outputs must be combined with partition functions:
\end{itemize}
\[
 o=\frac{Z_c o_c+Z_r o_r}{Z_c+Z_r}\quad\text{(not simply }o_c+o_r\text{).}
\]

\papersource{Sec. 3.2; PDF p. 6; Eqs. (10)--(12). Last equation: direct derivation.}
\end{frame}
\begin{frame}{Linear scaling requires a fixed memory budget}

\begin{itemize}
\item Main attention arithmetic: $O(Nmd+Nksd)$, including landmark probing.
\item Linear in $N$ only if $m,k,s,d$ are held fixed.
\item Top-$k$, query grouping, random gather, and fusion add real overhead.
\item Random K/V access can make the pipeline memory-bound.
\end{itemize}
\begin{block}{Changing the budget changes the claim}
If $m\approx k\propto\sqrt N$, the main arithmetic becomes $O(N^{3/2}d)$ for fixed $s$ (our analysis).
\end{block}

\papersource{Sec. 3.2, PDF p. 6; Algorithm 1, p. 16.}
\end{frame}
\begin{frame}{Local landmarks can retrieve nonlocal information}

\begin{columns}[c]
\column{0.58\textwidth}
\includegraphics[width=\linewidth,height=0.65\textheight,keepaspectratio]{figures/fig03-experts.pdf}
\column{0.39\textwidth}
\begin{itemize}
\item Red box: landmark pooling region.
\item Top: selected K/V.
\item Bottom: queries routed to that expert.
\item Neither set is confined to the box.
\end{itemize}
\end{columns}

\papersource{Original Figure 3; PDF p. 5. Heatmaps average attention heads.}
\end{frame}
\begin{frame}{Evaluation uses different budgets for different tasks}

\begin{center}
\begin{tabular}{llll}\toprule
Evaluation & Input & $(m,k,s)$ & Key metric\\\midrule
ImageNet-1K & $224^2$, patch 16 & $(25,25,1)$ & Accuracy\\
ADE20K & $512^2$ / $640^2$ & $(49,49,1)$ & mIoU\\
LRA & 1K--4K tokens & $(128,128,1)$ & Acc., steps/s\\
Speed curve & 1K--1024K & $mk\approx N/4$ & Samples/s\\\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item LRA and speed measurements: RTX 4090, 24 GB.
\item Standard attention uses a fused/FlashAttention implementation.
\item Speed curve: 3 layers, embedding 128; batch tuned per run.
\end{itemize}

\papersource{Secs. 4.1--4.3; PDF pp. 8--9; Figure 5 legend distinguishes the speed budget.}
\end{frame}
\begin{frame}{Basic MiTA nearly matches full attention on DeiT-S}

\begin{center}
\begin{tabular}{lrrr}\toprule
Attention / extras & Tiny acc. & Small acc. & Small FLOPs\\\midrule
Standard & 72.2 & 79.9 & 4.6G\\
Agent & 70.3 & 78.5 & 4.4G\\
Agent + bias & 71.1 & 79.1 & 4.4G\\
MiTA & 71.1 & 79.8 & 4.4G\\
MiTA + DWC & 73.4 & 80.6 & 4.4G\\
MiTA + DWC + gate & --- & 81.2 & 4.7G\\\bottomrule
\end{tabular}
\end{center}
\medskip
Base MiTA: $-1.1$ / $-0.1$ accuracy points vs. full attention.\\
The gains above full attention include \alert{extra architectural components}.

\papersource{Table 2, PDF p. 7; Sec. 4.1, p. 8. Accuracies in percent.}
\end{frame}
\begin{frame}{A stronger architecture improves MiTA, but preserves a gap}

\begin{center}
\begin{tabular}{lrr}\toprule
Model & Accuracy (\%) & FLOPs\\\midrule
ViT-5-S & 82.2 & 4.7G\\
DeiT-III-S & 81.4 & 4.6G\\
MiTA-ViT-5-S & 81.3 & 4.5G\\
MiTA-ViT-5-S + DWC & 81.7 & 4.5G\\\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Best listed MiTA variant is 0.5 points below ViT-5-S.
\item Fewer FLOPs, but no universal accuracy dominance.
\item This is a comparison of complete models, not an isolated attention ablation.
\end{itemize}

\papersource{Selected rows of Table 3, PDF p. 7.}
\end{frame}
\begin{frame}{Segmentation reduces FLOPs at a measurable accuracy cost}

\begin{center}
\begin{tabular}{lrrr}\toprule
Backbone & FLOPs: full $\to$ MiTA & mIoU: full $\to$ MiTA & $\Delta$ points\\\midrule
Tiny, $512^2$ & 12G $\to$ 7G & 39.1 $\to$ 36.5 & $-2.6$\\
Small, $512^2$ & 33G $\to$ 25G & 45.8 $\to$ 44.5 & $-1.3$\\
Base, $512^2$ & 111G $\to$ 95G & 49.3 $\to$ 48.1 & $-1.2$\\
Large, $640^2$ & 622G $\to$ 508G & 53.3 $\to$ 50.5 & $-2.8$\\\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Matched comparisons use the same CBT segmentation head.
\item 14--42\% fewer reported FLOPs; 1.2--2.8 mIoU points lost.
\item Backbones were pretrained with standard attention, not native MiTA.
\end{itemize}

\papersource{Table 4, PDF p. 8; Sec. 4.2, pp. 8--9.}
\end{frame}
\begin{frame}{LRA combines near-full accuracy with faster training}

\begin{center}
\begin{tabular}{lrrr}\toprule
Task & Full acc. & MiTA acc. & Steps/s speedup\\\midrule
ListOps (2K) & 37.35 & 37.20 & 4.1$\times$\\
Text (4K) & 63.63 & 63.47 & 9.9$\times$\\
Retrieval (4K) & 79.90 & 79.88 & 10.3$\times$\\
Image (1K) & 40.38 & 40.72 & 2.4$\times$\\
Pathfinder (1K) & 69.68 & 73.26 & 2.7$\times$\\\midrule
Mean accuracy & 58.19 & 58.91 & \\\bottomrule
\end{tabular}
\end{center}
Total training: 10.6 $\to$ 2.4 hours ($77.4\%$ less).\\
Accuracy gains are concentrated in Pathfinder, not uniform across tasks.

\papersource{Table 5 and Sec. 4.3, PDF p. 9. Accuracy in percent; same training recipe.}
\end{frame}
\begin{frame}{Compression helps MiTA, while Agent remains faster overall}

\begin{center}
\begin{tabular}{lrr}\toprule
Method & Mean accuracy (\%) & Total hours\\\midrule
Full attention & 58.19 & 10.6\\
Agent Attention & 58.22 & 1.9\\
MoBA, route-only & 56.74 & 4.6\\
MiTA, route-only & 58.81 & 5.5\\
MiTA, full & 58.91 & 2.4\\\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Route-only variants preserve the attended-entry budget by increasing $k$.
\item More gathered entries can cost more than a compact shared summary.
\item MiTA is more accurate than Agent here, but takes longer to train.
\end{itemize}

\papersource{Table 5, PDF p. 9. No multi-seed uncertainty is reported.}
\end{frame}
\begin{frame}{Long-context throughput gains are setting-specific}

\begin{columns}[c]
\column{0.55\textwidth}
\includegraphics[width=\linewidth,height=0.63\textheight,keepaspectratio]{figures/fig05-speed.pdf}
\column{0.42\textwidth}
\begin{itemize}
\item At 1024K: about 159$\times$ throughput.
\item 3-layer, width-128 model on RTX 4090.
\item Batch size optimized independently per run.
\item Legend: $mk\approx N/4$.
\end{itemize}
\end{columns}
\medskip
This does not establish fixed-batch latency or million-token task accuracy.

\papersource{Original Figure 5 and Sec. 4.3; PDF p. 9.}
\end{frame}
\begin{frame}{Ablations favor reusable wider experts}

\begin{columns}[t]
\column{0.48\textwidth}
\textbf{Landmark extraction (accuracy)}
\begin{itemize}
\item 2D average pooling: 71.1
\item Learned parameters: 66.3
\item Convolution: 67.0
\item Token merging: 71.3
\end{itemize}
\column{0.48\textwidth}
\textbf{Equal $mk$ budgets}
\begin{itemize}
\item $(16,25)$: 71.0
\item $(25,16)$: 70.5
\item $(25,36)$: 71.7
\item $(36,25)$: 71.2
\end{itemize}
\end{columns}
\medskip
Full: 71.1; compression-only: 70.3; routing-only: 70.8.\\
Gradient conflict is an author hypothesis, not directly measured evidence.

\papersource{Table 6, Appendix A; PDF pp. 14--15. Values are accuracy percentages.}
\end{frame}
\begin{frame}{The pruning effect is selective retrieval, not token deletion}

\begin{center}
\includegraphics[width=0.94\linewidth,height=0.43\textheight,keepaspectratio]{figures/fig04-pruning.pdf}
\end{center}
\begin{itemize}
\item Rows: training images; columns: input followed by successive layers.
\item Later experts select class-relevant regions more strongly.
\item Unselected tokens can still contribute through the global summary.
\end{itemize}

\papersource{Original Figure 4; PDF p. 7. Qualitative examples, aggregated over heads.}
\end{frame}
\begin{frame}{Fixed-weight swapping: the plotted data contradict the text}

\begin{columns}[c]
\column{0.46\textwidth}
\includegraphics[width=\linewidth,height=0.58\textheight,keepaspectratio]{figures/fig09-tiny.pdf}
\column{0.51\textwidth}
\textbf{Standard-trained models}\\MiTA at inference
\begin{itemize}
\item Tiny: $58.7/72.1=81.4\%$ retention.
\item Small: $71.3/80.0=89.1\%$ retention.
\item Appendix C claims over 95\% in this direction.
\end{itemize}
The figure and text disagree.
\end{columns}

\papersource{Figure 9 (Tiny panel shown), PDF p. 16; conflicting statement in Appendix C, p. 17.}
\end{frame}
\begin{frame}{Finetuning transfers better to MiTA than to Agent}

\begin{center}
\begin{tabular}{lrrrr}\toprule
Attention & Tiny & Small & Base & Large\\\midrule
Standard & 76.9 & 81.2 & 84.4 & 85.9\\
Linear & 73.2 & 78.6 & 79.8 & 80.7\\
Agent (49) & 74.6 & 79.7 & 81.4 & 83.5\\
Agent (64) & 74.7 & 79.9 & 81.5 & 83.8\\
MiTA & 75.6 & 80.9 & 82.8 & 85.5\\\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item ImageNet-21K initialization; 50 epochs of ImageNet-1K finetuning.
\item Standard-to-MiTA gaps remain 0.3--1.6 accuracy points.
\item This is adaptation with training, distinct from frozen-weight swapping.
\end{itemize}

\papersource{Table 7 and Appendix C, PDF p. 17. Accuracy in percent; AdamW optimizer.}
\end{frame}
\begin{frame}{What is established, and what remains open?}

\begin{enumerate}
\item \textbf{Established design}: reusable, content-adaptive K/V experts coupled to compressed global memory.
\item \textbf{Established tradeoff}: strong LRA training gains; classification and segmentation retain measurable accuracy gaps.
\item \textbf{Open extension}: causal construction, incremental caching, $s>1$, and LLM decoding.
\item \textbf{Reading caution}: fixed-budget scaling, pretrained-vs-native comparisons, and internal figure/text inconsistencies.
\end{enumerate}

\papersource{Secs. 3--4; Appendices A--D; PDF pp. 5--9, 14--17.}
\end{frame}
\begin{frame}{Reference and discussion prompts}

Qishuai Wen, Zhiyuan Huang, Wei He, Xianghan Meng, and Chun-Guang Li.\\
\emph{Mixture-of-Top-$k$ Attention: Efficient Attention via Scalable Fast Weights.}\\
arXiv:2602.01219v5, 9 May 2026.
\medskip

\href{https://arxiv.org/abs/2602.01219v5}{arxiv.org/abs/2602.01219v5}\qquad
\href{https://github.com/QishuaiWen/MiTA}{github.com/QishuaiWen/MiTA}
\medskip
\begin{itemize}
\item What budget allocation best preserves rare details?
\item Can experts and summaries be updated causally at low cost?
\item How much of the measured gain survives at matched quality and batch size?
\end{itemize}

\papersource{Reading based on the version-matched PDF, main.tex, and original figures. No code reproduction.}
\end{frame}
\begin{frame}{Backup: larger inference budgets can help or hurt}
\textbf{Fixed weights: MiTA-ViT-T at $512^2$ resolution}
\begin{center}
\begin{tabular}{rrrl}\toprule
$m$ & $k$ & Accuracy (\%) & Configuration\\\midrule
25 & 25 & 77.1 & Training baseline\\
25 & 36 & 77.6 & Wider experts\\
36 & 36 & 77.7 & More and wider experts\\
100 & 100 & 75.2 & Much larger inference budget\\\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Only inference hyperparameters change; no weight updates.
\item A modest increase helps in these selected cells.
\item A larger memory budget does not guarantee higher accuracy.
\end{itemize}
\papersource{Selected cells transcribed from Figure 10(a), PDF p. 16; Appendix C, p. 17.}
\end{frame}
\begin{frame}{Backup: routing is not identical to hard spatial clustering}

\begin{columns}[c]
\column{0.55\textwidth}
\includegraphics[width=\linewidth,height=0.60\textheight,keepaspectratio]{figures/fig08-overlap.pdf}
\column{0.42\textwidth}
\begin{itemize}
\item Compare gathered K/V positions with routed-query positions.
\item Vertical axis: mIoU, averaged over experts and heads.
\item Modest overlap supports nonlocal information flow.
\item It does not prove retrieval completeness or semantic correctness.
\end{itemize}
\end{columns}

\papersource{Original Figure 8; Appendix B; PDF p. 15.}
\end{frame}
\begin{frame}{Backup: two implementation details need explicit handling}

\begin{enumerate}
\item Compute expert labels $a=\arg\max(\tilde Q^\top Q)$.
\item Sort queries by $a$; retain the inverse permutation.
\item Build cumulative lengths from \alert{counts of expert labels}, not counts of the query permutation.
\item Run grouped attention; restore the original query order.
\item Combine branches using their log-sum-exp normalizers.
\end{enumerate}
\medskip
Algorithm 1 conflates the label-count and permutation variables and omits an explicit inverse permutation. This is a pseudocode observation, not a repository audit.

\papersource{Algorithm 1, PDF p. 16; corrections above are reading analysis.}
\end{frame}
\begin{frame}{Backup: $m=k=N$ does not generally recover full attention}

Appendix A makes this claim, but Eq. (10) still includes the shared branch.
\[
d=1,\quad Q=K=[1,0],\quad V=[1,0],\quad m=k=N=2,\quad s=1.
\]
For $q=1$, let $a=e/(e+1)\approx0.7311$. Then
\[
\tilde V=[a,0.5],\qquad
\att(q,K,V)=a,\qquad
\operatorname{MiTA}(q)=\frac{ea+0.5+e}{2e+2}\approx0.7000.
\]
\alert{Additional conditions or removal of the shared branch are needed.}\\
This is a constructed formula check, not a reported paper experiment.

\papersource{Claim: Appendix A, PDF p. 14. Formula: Eqs. (10)--(12), p. 6. Counterexample: our analysis.}
\end{frame}
\end{document}