\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\E}{\mathbb E}
\newcommand{\R}{\mathbb R}
\newcommand{\one}{\mathbf 1}
\newcommand{\diag}{\operatorname{diag}}
\title[RACE Attention]{RACE Attention}
\subtitle{A Strictly Linear-Time Attention Layer for\ Training on Outrageously Large Contexts}
\author{Paper: Sahil Joshi, Agniva Chowdhury, Amar Kanakamedala,\\Ekam Singh, Evan Tu, Anshumali Shrivastava}
\institute{Rice University\quad\textbar\quad Independent paper discussion}
\date{arXiv:2510.04008v5 (18 April 2026)\\Reading: 15 October 2026}

\begin{document}

\begin{frame}
\titlepage
\vspace{-4mm}
\begin{center}\color{AcademicTeal}Replace token-pair interactions with differentiable bucket summaries.\end{center}
\end{frame}

\begin{frame}{The paper asks two distinct scaling questions}
\begin{columns}[T]
\begin{column}{0.48\textwidth}
\begin{block}{Can the model preserve task quality?}
Train Transformers with RACE in place of softmax attention.
\end{block}
\begin{itemize}
\item Classification up to 64K tokens.
\item Causal language modeling at 128 and 1024 tokens.
\item Compare accuracy and perplexity.
\end{itemize}
\end{column}
\begin{column}{0.48\textwidth}
\begin{block}{Can the operator process huge inputs?}
Time one attention layer, one forward--backward pass.
\end{block}
\begin{itemize}
\item Up to 12M tokens on GH200.
\item Up to 75M tokens on CPU.
\item Batch 1, 4 heads, reported $d=128$.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Sec. 4, Tables 1--4 and Fig. 4, PDF pp. 8--11; Table 8, p. 16.}
\end{frame}

\begin{frame}{Choose a nonnegative kernel that can be sketched}
\[
O_i=\frac{\sum_j S_{ij}V_j}{\sum_j S_{ij}},\qquad
S_{ij}=\left(1-\frac{1}{\pi}\arccos\frac{Q_i^\top K_j}{\|Q_i\|\|K_j\|}\right)^\gamma.
\]
\begin{itemize}
\item The angular kernel is $1$, $1/2$, and $0$ at angles $0$, $\pi/2$, and $\pi$.
\item Raising it to $\gamma$ increases contrast: orthogonal pairs have score $2^{-\gamma}$.
\item Normalization makes each output a convex combination of values.
\end{itemize}
\begin{block}{Three objects to keep separate}
Softmax attention $\longrightarrow$ angular attention $\longrightarrow$ soft RACE approximation.
\end{block}
\papersource{Paper, Sec. 3.1, Eqs. (3)--(4), PDF pp. 5--6.}
\end{frame}

\begin{frame}{Powers sharpen the angular similarity pattern}
\centering
\includegraphics[width=0.97\linewidth,height=0.48\textheight,keepaspectratio]{figures/fig-03-angular-heatmaps.png}
\begin{itemize}
\item Rows: queries. Columns: keys. Angular panels use $\gamma=1,6,10,12$.
\item Larger powers concentrate the displayed high-similarity regions.
\item Color scales differ; visual similarity does not establish kernel equivalence.
\end{itemize}
\papersource{Original Fig. 3, PDF p. 6. Angular ignores vector norms; dot-product softmax does not.}
\end{frame}

\begin{frame}{A hash collision estimates a powered angular kernel}
For a Gaussian hyperplane $w\sim\mathcal N(0,I_d)$,
\[
\Pr[\operatorname{sign}(w^\top q)=\operatorname{sign}(w^\top k)]
=\kappa_{\rm ang}(q,k).
\]
Concatenate $P$ independent signs:
\[
h(x)=\operatorname{sign}(Wx)\in\{\pm1\}^P,
\qquad \Pr[h(q)=h(k)]=\kappa_{\rm ang}(q,k)^P.
\]
\begin{itemize}
\item $R=2^P$ possible buckets; the target exponent is \alert{$\gamma=P$}.
\item $L$ independent tables average repeated collision estimates.
\item Hard assignments obstruct ordinary backpropagation.
\end{itemize}
\papersource{Paper, Sec. 2.2, PDF p. 4; Sec. 3.2, p. 7.}
\end{frame}

\begin{frame}{Soft corner assignments make hashing differentiable}
\[
u^{(\ell)}(x)=\tanh(W^{(\ell)}x),\qquad
\phi_r^{(\ell)}(x)=
\frac{e^{\beta u^{(\ell)}(x)^\top v_r}}
{\sum_{r'}e^{\beta u^{(\ell)}(x)^\top v_{r'}}}.
\]
\begin{itemize}
\item $W^{(\ell)}\in\R^{P\times d}$ has independent Gaussian rows.
\item $v_r\in\{\pm1\}^P$ is one of $R$ hypercube corners.
\item Each token becomes a probability vector over a small bucket bank.
\item $\beta\to\infty$: hard hashing.\quad $\beta\to0$: uniform buckets.
\end{itemize}
\begin{block}{Training choice}
The experiments learn $\beta$; it multiplies logits and acts as an inverse temperature.
\end{block}
\papersource{Paper, Algorithm 1, PDF p. 5; trainable $\beta$, Sec. 4, p. 8.}
\end{frame}

\begin{frame}{Each bucket stores mass and a weighted value sum}
For table $\ell$, build $\Phi_K^{(\ell)},\Phi_Q^{(\ell)}\in\R^{N\times R}$.
\begin{columns}[T]
\begin{column}{0.48\textwidth}
\begin{block}{Mass}
\[
A^{(\ell)}=(\Phi_K^{(\ell)})^\top\one_N\in\R^R
\]
\[
A_r^{(\ell)}=\sum_j\phi_r^{(\ell)}(K_j)
\]
Needed for the final denominator.
\end{block}
\end{column}
\begin{column}{0.48\textwidth}
\begin{block}{Content}
\[
B^{(\ell)}=(\Phi_K^{(\ell)})^\top V\in\R^{R\times d}
\]
\[
B_r^{(\ell)}=\sum_j\phi_r^{(\ell)}(K_j)V_j
\]
Stores the weighted values.
\end{block}
\end{column}
\end{columns}
\medskip
Every key writes to the same small bank; queries read this bank.
\papersource{Paper, Algorithm 1, PDF p. 5; Sec. 3.2, p. 7.}
\end{frame}

\begin{frame}{Normalize after combining the tables}
\[
\mathrm{Num}=\frac1L\sum_\ell\Phi_Q^{(\ell)}B^{(\ell)},\qquad
\mathrm{Den}=\frac1L\sum_\ell\Phi_Q^{(\ell)}A^{(\ell)}
\]
\[
\widehat O=\diag(\mathrm{Den})^{-1}\mathrm{Num}.
\]
\begin{block}{The implicit kernel}
\[
\widehat S_{ij}=\frac1L\sum_\ell\phi^{(\ell)}(Q_i)^\top\phi^{(\ell)}(K_j).
\]
Associate $\Phi_Q(\Phi_K^\top V)$ to avoid forming $\widehat S\in\R^{N\times N}$.
\end{block}
\begin{itemize}
\item Ratio of summed numerators and denominators, rather than an average of ratios.
\end{itemize}
\papersource{Paper, Algorithm 1, PDF p. 5. See backup for a numerical example and Fig. 7 caveat.}
\end{frame}

\begin{frame}{Causality is a prefix update of the same statistics}
Treat $\phi$ and $V_t$ as column vectors:
\[
A_t^{(\ell)}=A_{t-1}^{(\ell)}+\phi^{(\ell)}(K_t),\qquad
B_t^{(\ell)}=B_{t-1}^{(\ell)}+\phi^{(\ell)}(K_t)V_t^\top.
\]
\[
\widehat O_t=
\frac{\sum_\ell\phi^{(\ell)}(Q_t)^\top B_t^{(\ell)}}
{\sum_\ell\phi^{(\ell)}(Q_t)^\top A_t^{(\ell)}}.
\]
\begin{itemize}
\item Update before reading: the current key/value is included.
\item Recurrent state: $LR(d+1)$ scalars for fixed feature maps.
\item Training activations and backward memory are additional costs.
\item The paper implements prefix scans using OpenMP/CUDA.
\end{itemize}
\papersource{Paper, Algorithm 2, PDF p. 27; implementation discussion, p. 28.}
\end{frame}

\begin{frame}{Linear cost assumes a fixed bucket budget}
\[
T=O\big(LN(dP+PR+Rd)\big)\simeq O(LNRd),\qquad R=2^P.
\]
\begin{columns}[T]
\begin{column}{0.42\textwidth}
\centering
\begin{tabular}{rrr}
\toprule
$(P,L)$ & $R$ & $LR$\\
\midrule
$(2,2)$ & 4 & 8\\
$(3,3)$ & 8 & 24\\
$(4,4)$ & 16 & 64\\
\bottomrule
\end{tabular}
\end{column}
\begin{column}{0.54\textwidth}
\begin{itemize}
\item Core sketch workspace: $O(L(NR+Rd))$.
\item Token inputs/outputs still cost $O(Nd)$.
\item QKV projections and FFNs retain their own costs.
\end{itemize}
\end{column}
\end{columns}
\begin{block}{Reader's deduction}
The non-causal kernel has rank at most $LR$. Fixed state creates a capacity trade-off.
\end{block}
\papersource{Paper, Sec. 3.2, PDF p. 7. Rank bound follows from the concatenated feature factorization.}
\end{frame}

\begin{frame}{The claimed bound separates bias and sampling error}
\[
\|\widehat O-O\|_{\rm rms}
=O\!\left(\frac P\beta+\sqrt{\frac{\log(N/\delta)}L}\right)\|V\|_F.
\]
\begin{itemize}
\item Target: $P$-powered angular attention in the \alert{non-causal} setting.
\item Appendix assumes unit Q/K and row mass $s_{\min}\ge C_1N$.
\item Bias falls with $\beta$; the simplified variance term uses $L\ge\log(2N/\delta)$.
\item Taking $L\propto\log N$ changes fixed-budget linear time to $O(N\log N\,Rd)$.
\end{itemize}
\begin{block}{Proof caveat from this reading}
Lemma 10 centers a biased soft estimator at $S$. Normalization control needs a bias condition as well as a table-count condition.
\end{block}
\papersource{Theorem 2, p. 8; Sec. 6.1, p. 17; Lemma 10, p. 24; simplification, p. 27. Backups give details.}
\end{frame}

\begin{frame}{The evaluation spans kernels, tasks, and hardware}
\begin{tabular}{p{0.26\linewidth}p{0.65\linewidth}}
\toprule
Comparison & Role\\
\midrule
FlashAttention2/3 & Exact softmax; optimized GPU execution\\[2pt]
Linear / Performer-256 & Simple positive / random feature maps\\[2pt]
Linformer-128 & Sequence projection approximation\\[2pt]
Angular ($\gamma=8$) & Alternative kernel, still quadratic pairwise evaluation\\[2pt]
YOSO-16 & Related hard-hash estimator\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Accuracy experiments: single A100, best-validation checkpoint.
\item Metrics: classification accuracy $\uparrow$; language-model perplexity $\downarrow$.
\item Tiny Stories uses masked LM; WikiText-103 and PTB use causal LM.
\end{itemize}
\papersource{Paper, Sec. 4, PDF pp. 8--10; Table 5, p. 15.}
\end{frame}

\begin{frame}{A moderate sketch can preserve task performance}
\centering
\begin{tabular}{lrrr}
\toprule
Method & CIFAR-10 & QNLI & Tiny Stories\\
 & Acc.\% $\uparrow$ & Acc.\% $\uparrow$ & PPL $\downarrow$\\
\midrule
RACE (2,2) & 63.7 & 60.7 & 4.2\\
RACE (3,3) & 62.5 & 60.7 & 3.2\\
RACE (4,4) & \textbf{65.9} & \textbf{61.1} & \textbf{2.6}\\
FlashAttention2 & 61.44 & \textbf{61.1} & 2.7\\
Linear & 60.0 & 60.7 & 7.0\\
Performer-256 & 64.9 & 61.0 & 10.0\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Contexts: CIFAR-10 1024, QNLI 2048, Tiny Stories 1024.
\item The smallest RACE budget loses quality on Tiny Stories.
\end{itemize}
\papersource{Selected rows from Table 2, PDF p. 9. All values are reported results.}
\end{frame}

\begin{frame}{Causal RACE matches the reported WikiText baseline}
\begin{columns}[T]
\begin{column}{0.52\textwidth}
\centering
\begin{tabular}{lr}
\toprule
WikiText-103 @1024 & PPL $\downarrow$\\
\midrule
RACE (2,2) & 23.9\\
RACE (2,3) & 23.4\\
RACE (3,3) & 21.9\\
RACE (3,4) & 21.5\\
RACE (4,4) & \textbf{20.9}\\
FlashAttention2 & \textbf{20.9}\\
Angular ($\gamma=8$) & 19.0\\
\bottomrule
\end{tabular}
\end{column}
\begin{column}{0.43\textwidth}
\begin{block}{PTB @128}
RACE (4,4): \textbf{53.4} PPL\\
FlashAttention2: 55.4 PPL
\end{block}
\begin{itemize}
\item More tables help in the shown fixed-$P$ WikiText comparisons.
\item Angular has a different exponent and quadratic computation.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Table 4, PDF p. 9; Table 8, p. 16.}
\end{frame}

\begin{frame}{At 64K, RACE trains in 584 s versus 1645 s}
\centering
\begin{tabular}{lrrrr}
\toprule
Arxiv & RACE Acc.\% & FA2 Acc.\% & RACE train & FA2 train\\
\midrule
16K & 71.3 & 69.8 & 82.4 s & 95.7 s\\
32K & 90.6 & 89.7 & 289 s & 471 s\\
64K & 97.92 & 97.00 & 584 s & 1645 s\\
\bottomrule
\end{tabular}
\par\medskip
Fixed RACE $(P,L)=(3,3)$; times are per epoch on A100 40 GB.
\begin{block}{64K comparison}
\textbf{+0.92 percentage points}; FA2/RACE training-time ratio \textbf{2.82$\times$}.
\end{block}
\begin{itemize}
\item Same-class documents are concatenated to build each example.
\item Dataset sizes differ by length; the rising accuracy is not a controlled context-length ablation.
\end{itemize}
\papersource{Table 1, PDF p. 9; preprocessing, pp. 15--16. Ratio calculated from reported epoch times.}
\end{frame}

\begin{frame}{Food-101 supports a same-batch comparison with FA2}
\centering
\begin{tabular}{lrrrr}
\toprule
Method & Batch & Train/epoch & Test/epoch & Acc.\%\\
\midrule
RACE (2,2) & 8 & 891 s & 37 s & 42.4\\
RACE (3,3) & 8 & 950 s & 40 s & \textbf{43.5}\\
RACE (4,4) & 8 & 1042 s & 42 s & 40.3\\
FlashAttention2 & 8 & 2600 s & 95 s & 42.1\\
Performer-256 & 1 & 2546 s & 105 s & 42.4\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item 51-class subset: 20,000 training / 2500 test images.
\item $512\times512$ images, patch/stride 4: 16,384 tokens.
\item RACE (3,3) vs. FA2: +1.4 points and 2.74$\times$ training-time ratio.
\item Linear, Linformer, and Performer used batch 1 after OOM at 8.
\end{itemize}
\papersource{Table 3 and batch-size exception, PDF p. 9; dataset construction, p. 16.}
\end{frame}

\begin{frame}{The GPU operator reaches 12M tokens at a small budget}
\begin{columns}[c]
\begin{column}{0.65\textwidth}
\includegraphics[width=\linewidth,height=0.64\textheight,keepaspectratio]{figures/fig-04d-gpu-scaling.png}
\end{column}
\begin{column}{0.32\textwidth}
\textbf{GH200, 96 GB}\\[3mm]
One layer; forward + backward.\\[3mm]
Maximum shown:
\begin{itemize}
\item (2,2): 12M
\item (3,3): 10M
\item (4,4): 7M
\end{itemize}
\end{column}
\end{columns}
\medskip
At $\sim$4M: authors report $\sim$0.1 s for RACE vs. $\sim$550 s for FA2.
\papersource{Original Fig. 4d and endpoints in Fig. 4a--c, PDF p. 10; reported timing, p. 11. Operator timings.}
\end{frame}

\begin{frame}{Algorithmic savings can outweigh GPU acceleration}
\begin{columns}[c]
\begin{column}{0.64\textwidth}
\includegraphics[width=\linewidth,height=0.62\textheight,keepaspectratio]{figures/fig-04e-cpu-vs-gpu.png}
\end{column}
\begin{column}{0.33\textwidth}
\begin{itemize}
\item RACE: Xeon Gold 5220R CPU.
\item Baselines: GH200 GPU.
\item Crossover near 131K.
\item CPU RACE reaches 75M, roughly 100 s.
\end{itemize}
\end{column}
\end{columns}
\medskip
At $\sim$4M, reported CPU-RACE speedup: $\sim$40$\times$ vs. GPU-FA2, $\sim$20$\times$ vs. GPU-FA3.
\papersource{Original Fig. 4e, PDF p. 10; Sec. 4, p. 11. The separate CPU 33M ``under 10 s'' claim conflicts with plotted CPU curves and is not used here.}
\end{frame}

\begin{frame}{More buckets do not guarantee better trained models}
\centering
\begin{tabular}{llrr}
\toprule
Task & Metric & Configuration 1 & Configuration 2\\
\midrule
Tiny Stories @512 & PPL $\downarrow$ & (5,4): \textbf{2.7} & (5,5): 5.1\\
FashionMNIST @784 & Acc.\% $\uparrow$ & (2,5): \textbf{87.7} & (4,5): 85.7\\
Food-101 @16K & Acc.\% $\uparrow$ & (3,3): \textbf{43.5} & (4,4): 40.3\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item $P$ changes both target-kernel sharpness and the exponential bucket count.
\item $L$ changes sampling budget and the model's training computation.
\item A fixed-input approximation bound does not imply monotonic trained-task quality.
\end{itemize}
\papersource{Paper, Table 3, PDF p. 9; Tables 6--7, p. 16.}
\end{frame}

\begin{frame}{Additional tasks show mostly modest accuracy gains}
\centering
\begin{tabular}{lrrr}
\toprule
Task & Best reported RACE & FA2 & Difference (points)\\
\midrule
ListOps @2000 & 41.9 & 41.4 & +0.5\\
Text Retrieval @8000 & 80.9 & 80.5 & +0.4\\
Yahoo @256 & 67.2 & 67.2 & 0.0\\
IMDB @512 & 81.3 & 80.0 & +1.3\\
SST-2 @1024 & 79.4 & 78.5 & +0.9\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Accuracy in percent; best RACE configuration selected separately per task.
\item ListOps uses (2,2); Retrieval/Yahoo/SST-2 use (4,4).
\item YOSO-16: 41.0 on ListOps and 80.2 on Retrieval.
\end{itemize}
\papersource{Paper, Tables 9--10, PDF p. 16.}
\end{frame}

\begin{frame}{RACE offers a concrete design for compact attention state}
\begin{enumerate}
\item \textbf{Mechanism:} differentiable angular hashing, bucket mass/value sums, and query-dependent normalization.
\medskip
\item \textbf{Evidence:} competitive task scores in the tested settings, with strong single-layer scaling at long sequences.
\medskip
\item \textbf{Trade-off:} finite bucket capacity, exponentially growing $2^P$, and non-monotonic training outcomes.
\medskip
\item \textbf{Theory:} useful bias--variance decomposition; normalization proof needs correction, and causal guarantees remain open.
\end{enumerate}
\papersource{Paper, Secs. 3--5 and Appendix 6. Proof caveat and rank interpretation are from this reading.}
\end{frame}

\begin{frame}{Discussion: connect the operator to a full long-context model}
\begin{itemize}
\item Can the denominator analysis be repaired with explicit soft-bias conditions?
\medskip
\item What happens at matched state capacity and matched target exponent?
\medskip
\item How much operator speedup survives full causal training and decoding?
\end{itemize}
\begin{block}{Primary source}
S. Joshi, A. Chowdhury, A. Kanakamedala, E. Singh, E. Tu, A. Shrivastava.\par
\emph{RACE Attention: A Strictly Linear-Time Attention Layer for Training on Outrageously Large Contexts}. arXiv:2510.04008v5.
\end{block}
\href{https://arxiv.org/abs/2510.04008v5}{Versioned paper}\quad\textbar\quad
\href{https://github.com/sahiljoshi515/RACE_Attention}{Author repository}
\papersource{Main presentation ends here. The following slides provide derivations and protocol details.}
\end{frame}

\appendix

\begin{frame}{Backup: the proof propagates kernel error through a ratio}
\[
\Delta=\widehat S-S
=\frac1L\sum_\ell(\widehat S^{(\ell)}-\E\widehat S^{(\ell)})+\widetilde B.
\]
\[
\|\Delta\|_2\le\|\widetilde B\|_2
+\frac{4N}{\sqrt L}\sqrt{\log\frac{2N}{\delta}}
+\frac{4N}{3L}\log\frac{2N}{\delta}.
\]
Let $D=\diag(S\one)$, $\widehat D=D+E$, $A=D^{-1}S$.
\[
\widehat A-A=\widehat D^{-1}\Delta+(\widehat D^{-1}-D^{-1})S.
\]
\begin{itemize}
\item Soft-hash bias: $\|\widetilde B\|_2=O(NP/\beta)$, plus an exponential tail.
\item Denominator stability requires $\|E\|_2\le s_{\min}/2$.
\item Then $\|\widehat O-O\|_F\le\|\widehat A-A\|_2\|V\|_F$.
\end{itemize}
\papersource{Theorem 6, pp. 19--20; Lemma 8, pp. 21--23; Lemma 11, p. 25; Theorem 13, p. 26.}
\end{frame}

\begin{frame}{Backup: Lemma 10 omits the soft-hash bias}
Define $b=\max_{ij}|\E\widehat S_{ij}-S_{ij}|$. With probability at least $1-\delta$,
\[
\frac{\|E\|_2}{N}\le b+\sqrt{\frac{\log(2N^2/\delta)}{2L}}.
\]
\begin{itemize}
\item Lemma 10 uses $b=0$, contradicting the preceding soft-bias analysis.
\item A sufficient repair is $b\le C_1/4$ and $L\ge(8/C_1^2)\log(2N^2/\delta)$.
\end{itemize}
\begin{block}{Counterexample to the lemma as written}
All Q/K equal one unit vector; $P=2$, $\beta=0.1$. Then $S_{ij}=1$, but
\[
\widehat S^{(\ell)}_{ij}=\|\phi^{(\ell)}(x)\|_2^2\le\max_r\phi^{(\ell)}_r(x)\le e^{0.4}/4<0.373.
\]
After averaging, $\|E\|_2>0.627N>N/2$ for every $L$.
\end{block}
\papersource{Reader's analysis of Lemma 10, PDF p. 24. This refutes that lemma's condition, not every output bound.}
\end{frame}

\begin{frame}{Backup: bucket masses change the answer}
Teaching example: one table, two buckets, two scalar values.
\[
\phi(K_1)=(0.8,0.2),\quad\phi(K_2)=(0.1,0.9),\quad V=(2,10).
\]
\[
A=(0.9,1.1),\qquad B=(2.6,9.4),\qquad\phi(Q)=(0.75,0.25).
\]
\[
\widehat O=\frac{0.75(2.6)+0.25(9.4)}{0.75(0.9)+0.25(1.1)}
=\frac{4.3}{0.95}\approx4.5263.
\]
\begin{itemize}
\item Equivalent pairwise weights: $(0.65,0.30)$ before normalization.
\item Mixing per-bucket means instead gives $4.3030$, a different operator.
\end{itemize}
\papersource{Teaching example derived from Algorithm 1, PDF p. 5. Fig. 7, p. 28 depicts within-bucket normalization; use Algorithm 1 for the precise global-ratio definition.}
\end{frame}

\begin{frame}{Backup: task models differ from the scaling operator}
\centering
\begin{tabular}{lrrrrr}
\toprule
Dataset & Layers & Heads & Config.\ $d$ & Batch & Epochs\\
\midrule
WikiText-103 & 8 & 8 & 512 & 16 & 100\\
PTB & 1 & 2 & 128 & 16 & 70\\
Tiny Stories & 6 & 4 & 384 & 32 & 150\\
Arxiv & 4 & 4 & 256 & 2 & 100\\
Food-101 & 8 & 8 & 512 & 8 & 100\\
Text Retrieval & 4 & 2 & 384 & 1 & 20\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item $d$ here is the model configuration reported in Table 5.
\item Arxiv/Food use gradient accumulation 16/4.
\item AdamW; usually 1\% warmup then linear decay. Constant LR below $2\times10^{-4}$.
\item Food baseline batch exceptions are shown in the main results slide.
\end{itemize}
\papersource{Paper, Table 5 and optimizer discussion, PDF p. 15.}
\end{frame}

\begin{frame}{Backup: fixed feature width explains dimension scaling}
\begin{columns}[c]
\begin{column}{0.62\textwidth}
\includegraphics[width=\linewidth,height=0.62\textheight,keepaspectratio]{figures/fig-06d-dimension-scaling.png}
\end{column}
\begin{column}{0.35\textwidth}
Fixed $N=65536$.\\[3mm]
Generic feature attention:
\[
O(Nmd).
\]
$m=d$ gives $O(Nd^2)$.\\[3mm]
RACE uses $m=LR$, fixed by its sketch budget.
\end{column}
\end{columns}
\medskip
This comparison does not imply every random-feature method must be quadratic in $d$.
\papersource{Original Fig. 6d, PDF p. 17. General feature-width interpretation is the reader's deduction.}
\end{frame}

\end{document}