% Paper discussion deck; compile with the included build_slides.py and pdfLaTeX.
% Main talk: frames 1--22 (about 25 minutes). Frames 23--30 are backup material.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{MemoryPurple}{RGB}{123,50,148}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\fig}[3]{\centering\includegraphics[width=#2\linewidth,height=#3\textheight,keepaspectratio]{figures/#1.pdf}\par}
\newcommand{\C}{\mathcal C}
\newcommand{\M}{\mathcal M}
\newcommand{\N}{\mathcal N}
\newcommand{\R}{\mathbb R}
\newcommand{\softmax}{\operatorname{softmax}}
\newcommand{\ppl}{\mathrm{PPL}}
\pdfstringdefDisableCommands{\def\newline{ }\def\translate#1{#1}}
\title[Shared memory in looped models]{The Surprising Effectiveness of Shared Memory in Looped Transformers}
\subtitle{LPT: learn to write once, then read across recursions}
\author{Paper by Giovanni Monea, Keshav Ramji, Yousef El-Kurdi,\newline Luis A. Lastras, Yoav Artzi, Nathan Godey,\newline and Ram\'on Fernandez Astudillo}
\institute{Cornell University and IBM Research\medskip\newline Independent paper-reading presentation}
\date{arXiv:2610.02383v1 \quad $\vert$ \quad 16 October 2026}

\begin{document}

\begin{frame}
\titlepage
\end{frame}

\begin{frame}{Training to share memory improves quality}
\begin{block}{The central result}
Later recursions reuse the first recursion's KV cache and keep only a short local window. Sharing is learned during pretraining.
\end{block}
\medskip
\centering
\begin{tabular}{lrrr}
\toprule
1B model & PPL $\downarrow$ & Accuracy $\uparrow$ & Memory / STD $\downarrow$\\
\midrule
STD, one pass & 11.35 & 62.5\% & 1.00\\
LT, five recursions & 11.39 & 63.4\% & 5.00\\
LPT, five recursions & 10.32 & 68.2\% & 1.06\\
H-LPT, five recursions & \textbf{10.23} & 68.1\% & \textbf{0.24}\\
\bottomrule
\end{tabular}
\papersource{Paper, Table 1, PDF p. 4. FineWeb-Edu PPL; mean accuracy over five zero-shot tasks.}
\end{frame}

\begin{frame}{Weight sharing does not eliminate cache growth}
\[
 h^{(t)}=\M^L(h_{\mathrm{in}}^{(t)}),\qquad
 h_{\mathrm{in}}^{(t+1)}=\N_{\mathrm{rec}}(h^{(t)})
\]
\begin{itemize}
\item One stack of $L$ layers is reused $R$ times: effective depth $LR$.
\item Each use receives a different hidden state and writes different keys and values.
\item Standard LT stores a full cache at every recursion:
\end{itemize}
\[
 \mathcal A_{\mathrm{LT}}^{(t)}(i)=\C_{\le i}^{(t)},
 \qquad M_{\mathrm{KV}}^{\mathrm{LT}}=LRT.
\]
\alert{Compute and context memory both grow with recurrent depth.}
\papersource{Paper, Sec. 2, PDF pp. 2--3, Eqs. (1)--(2); Eq. (4), p. 4. Common KV byte factors omitted.}
\end{frame}

\begin{frame}{LPT assigns memory and prediction to different passes}
\begin{columns}[c,onlytextwidth]
\column{0.40\textwidth}
\fig{fig01-architecture}{0.97}{0.79}
\column{0.57\textwidth}
\begin{itemize}
\item \textbf{State recursion:} build a full causal KV cache at each layer.
\medskip
\item \textbf{Prediction recursions:} read that same layer's shared cache and a local window.
\medskip
\item Keep the sequential hidden-state recurrence and shared weights.
\end{itemize}
\medskip
Sharing is \alert{across recursions at a fixed layer}.
\end{columns}
\papersource{Paper, Fig. 1 (left), PDF p. 1; Sec. 3, p. 3. Original panel, including its legend.}
\end{frame}

\begin{frame}{Shared and local entries compete in one softmax}
\[
\mathcal A_{\mathrm{LPT}}^{(t)}(i)=
\begin{cases}
\C_{\le i}^{(1)}, & t=1,\\[2pt]
\C_{\le i}^{(1)}\cup\C_{[i-\omega,i]}^{(t)}, & t=2,\ldots,R.
\end{cases}
\]
\[
o=\softmax\!\left(\frac{q[K_s;K_l]^\top}{\sqrt{d_h}}+\mathrm{mask}\right)[V_s;V_l]
\]
\begin{itemize}
\item $s$: shared entries; $l$: local entries; $d_h$: head dimension.
\item $\omega$ counts \emph{preceding} positions; the current local entry also participates.
\item $\omega=0$: no local history. $\omega=T$: full local history plus sharing.
\end{itemize}
\papersource{Paper, Sec. 3, PDF p. 3, Eq. (3). Second equation is a teaching expansion of the joint softmax.}
\end{frame}

\begin{frame}{Five recursions need only 6.25\% more cache than STD}
\[
M_{\mathrm{KV}}^{\mathrm{LT}}=LRT,\qquad
M_{\mathrm{KV}}^{\mathrm{LPT}}=L[T+(R-1)\omega]
\]
\centering
\begin{tabular}{lrr}
\toprule
$T=4096,\ R=5,\ \omega=64$ & LT & LPT\\
\midrule
Cached positions per layer & 20,480 & 4,352\\
Cache relative to STD & $5.00\times$ & $1.0625\times$\\
\bottomrule
\end{tabular}
\medskip
\begin{block}{Storage and reading are different costs}
LPT retains 21.25\% of LT's cache ($4.706\times$ smaller), but every recursion still reads a full shared prefix.
\end{block}
Local windows add 1.25\% attended entries; total decoding FLOPs rise by less than 1\% relative to LT.
\papersource{Paper, Sec. 3, PDF p. 4, Eq. (4); Table 1. Steady-state cache convention; current-entry details omitted.}
\end{frame}

\begin{frame}{H-LPT combines cache sharing with a hybrid backbone}
\begin{columns}[t,onlytextwidth]
\column{0.48\textwidth}
\begin{block}{LT2 baseline}
Gated DeltaNet layers, with roughly one full-attention layer per five layers.
\end{block}
\begin{itemize}
\item GDN state is independent of context length.
\item Remaining attention caches still grow with $R$.
\end{itemize}
\column{0.48\textwidth}
\begin{block}{H-LPT modification}
Apply the LPT sharing rule to each full-attention layer.
\end{block}
\begin{itemize}
\item GDN keeps a separate state per recursion.
\item Full-attention caches share the first recursion's memory.
\end{itemize}
\end{columns}
\medskip
\alert{Context memory includes both KV caches and GDN recurrent states.}
\papersource{Paper, Secs. 2--4, PDF pp. 3--5; Appendix A, p. 15.}
\end{frame}

\begin{frame}{Experiments hold parameters and training tokens fixed}
\centering
\begin{tabular}{lrrr}
\toprule
Base configuration & S & M & L\\
\midrule
Parameters & 151.9M & 454.2M & 1.01B\\
Layers / hidden width & 12 / 768 & 24 / 1024 & 36 / 1280\\
FineWeb-Edu tokens & 20B & 20B & 50B\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Context: 4,096; recursions: 2, 3, 5; default window: 64.
\item PPL: 10M held-out tokens. Zero-shot accuracy: ARC-Easy, HellaSwag, PIQA, SciQ, and LAMBADA (mean).
\item LT uses learned weights over per-pass NLLs (\textbf{adaptive}); others supervise only the last pass (\textbf{final}).
\item Same optimizer recipe; four seeds at S, one main run at M/L.
\end{itemize}
\papersource{Paper, Sec. 4, PDF pp. 4--5; Appendices A--B, pp. 15--16. More recursions require more compute.}
\end{frame}

\begin{frame}{LPT keeps improving from three to five recursions}
\centering
\begin{tabular}{llrrr}
\toprule
Size & Model & $R=2$ PPL & $R=3$ PPL & $R=5$ PPL\\
\midrule
S & LT & 16.17 & 15.98 & 16.00\\
  & LPT & 15.73 & 15.28 & \textbf{14.91}\\
M & LT & 13.28 & 13.13 & 13.34\\
  & LPT & 12.91 & 12.55 & \textbf{12.33}\\
L & LT & 11.20 & 11.11 & 11.39\\
  & LPT & 10.75 & 10.45 & \textbf{10.32}\\
\bottomrule
\end{tabular}\par
\medskip
At five recursions, LPT improves accuracy over LT by\newline
\alert{2.5 / 3.7 / 4.8 percentage points} at S / M / L.
\papersource{Paper, Table 1, PDF p. 4. Main objectives: adaptive for LT, final for LPT. PPL has no nats unit.}
\end{frame}

\begin{frame}{H-LPT reaches high accuracy with much less context memory}
\fig{fig02-frontier}{0.94}{0.60}
\medskip
At $R=5$, relative to same-size STD across S/M/L:
\begin{itemize}
\item PPL lower by 1.12--1.82; accuracy higher by 3.1--5.6 points.
\item Context memory lower by approximately 76--79\%.
\end{itemize}
\papersource{Paper, Fig. 2, PDF p. 5; Table 1, p. 4. Curves: $R=2,3,5$; largest marker: $R=5$.}
\end{frame}

\begin{frame}{Sharing makes final-only supervision effective}
\fig{fig03-supervision}{0.98}{0.48}
\medskip
\begin{itemize}
\item Under the default Ouro normalization, final-loss LT collapses at $R=5$.
\item LPT stays stable; final loss beats its adaptive loss by 0.16--0.50 PPL.
\item Interpretation: intermediate passes can specialize in further computation.
\end{itemize}
\papersource{Paper, Fig. 3, PDF p. 7; Sec. 6, p. 6. Filled: final; hollow: adaptive. Upper strip uses a log scale.}
\end{frame}

\begin{frame}{Access to shared memory matters more than window size}
\fig{fig04-window}{0.98}{0.46}
\medskip
\begin{itemize}
\item LPT beats LT at every tested window, including $\omega=0$.
\item With full local history, gains remain: 0.24--1.15 PPL.
\item $\omega=64$ balances quality and cache size.
\end{itemize}
\papersource{Paper, Fig. 4, PDF p. 7; Sec. 6, p. 6. Negative PPL differences favor LPT. LT uses its better objective.}
\end{frame}

\begin{frame}{Later recursions place more attention on shared memory}
\fig{fig05-attention}{0.98}{0.48}
\medskip
\begin{itemize}
\item Shared attention mass: 57--71\% in earlier prediction passes.
\item Final recursion: 73--80\%; mean over heads and query positions.
\item Full local caches: overall preference persists, with some exceptions.
\end{itemize}
\papersource{Paper, Fig. 5, PDF p. 8; Fig. 11, p. 19. Same 4,096-token sequence; mean rows use all layers.}
\end{frame}

\begin{frame}{Shared and local keys learn different representations}
\fig{fig06-pca}{0.98}{0.62}
\medskip
\begin{itemize}
\item LPT separates first-pass keys from later-pass keys more clearly.
\item Joint Q/K PCA before RoPE; each panel has its own basis.
\end{itemize}
\papersource{Paper, Fig. 6, PDF p. 8; Appendix D, p. 17. S, $R=3$, final loss for both models.}
\end{frame}

\begin{frame}{Shared KV gives the first pass a direct gradient route}
\fig{fig07-gradient}{0.95}{0.61}
\medskip
\begin{itemize}
\item LT's first-pass gradient shrinks strongly with depth.
\item LPT restores signal; shared KV dominates its first-pass contribution at $R=5$.
\end{itemize}
\papersource{Paper, Fig. 7, PDF p. 9. First LPT bar: path projections. Whiskers: 95\% sequence-bootstrap intervals.}
\end{frame}

\begin{frame}{Detaching shared reads harms learning from scratch}
\centering
\begin{tabular}{llrr}
\toprule
Size & Model, final loss & $R=3$ PPL & $R=5$ PPL\\
\midrule
S & LT & 16.32 & 2,040\\
  & LPT, detached shared KV & 16.14 & 33.21\\
  & LPT & \textbf{15.28} & \textbf{14.91}\\
M & LT & 31.74 & 2,090\\
  & LPT, detached shared KV & 1,995 & 1,527\\
  & LPT & \textbf{12.55} & \textbf{12.33}\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Forward reads and the hidden-state gradient path remain unchanged.
\item Evidence supports a learning role for shared-cache gradients.
\end{itemize}
\papersource{Paper, Table 6, PDF p. 20; Fig. 8, p. 9. Does not separate path length from gradient scale/direction.}
\end{frame}

\begin{frame}{Stronger normalization narrows the gap to LT}
\centering
\begin{tabular}{llrrr}
\toprule
Size & Model & $R=2$ PPL & $R=3$ PPL & $R=5$ PPL\\
\midrule
S & Best tested LT norm & 16.04 & 15.65 & 15.24\\
  & LPT, Ouro norm & 15.73 & 15.28 & 14.91\\
M & Best tested LT norm & 12.96 & 12.75 & 12.76\\
  & LPT, Ouro norm & 12.91 & 12.55 & 12.33\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Alternatives allow final-loss LT to train at depth.
\item LPT stays ahead, but the M / $R=2$ displayed gap is only 0.05 PPL.
\item Stability claims depend on normalization and optimizer settings.
\end{itemize}
\papersource{Paper, Table 5, PDF p. 17. Best LT selected separately per setting; learning rate/warmup not retuned.}
\end{frame}

\begin{frame}{Memory savings improve capacity while recursions add latency}
\centering
\begin{tabular}{lrrrr}
\toprule
1B model & $R$ & Max batch & Tokens/s & B=1 ms/step\\
\midrule
STD & 1 & 108 & 3,162 & 6.7\\
LT & 5 & 21 & 303 & 37.9\\
LPT & 5 & 101 & 578 & 42.3\\
LT2 & 5 & 103 & 1,776 & 21.7\\
H-LPT & 5 & 390 & 2,553 & 22.6\\
H-LPT & 3 & 444 & 4,333 & 13.6\\
\bottomrule
\end{tabular}\par
\medskip
LPT vs. LT, $R=5$: $4.81\times$ batch capacity, $1.91\times$ peak throughput, but $1.12\times$ batch-one latency.
\papersource{Paper, Table 8, PDF p. 25. One H100 80GB; bf16, 4,096 context, shared research implementation.}
\end{frame}

\begin{frame}{Nine recursions expose a limit of pure-attention LPT}
\centering
\begin{tabular}{lrrrr}
\toprule
 & \multicolumn{2}{c}{LPT PPL $\downarrow$} & \multicolumn{2}{c}{H-LPT PPL $\downarrow$}\\
\cmidrule(lr){2-3}\cmidrule(lr){4-5}
Size & $R=5$ & $R=9$ & $R=5$ & $R=9$\\
\midrule
S & 14.91 & 14.80 & 15.01 & 14.74\\
M & 12.33 & 12.39 & 12.31 & 12.14\\
L & 10.32 & 10.87 & 10.23 & 10.11\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item 1B LPT accuracy falls from 68.2\% to 66.5\%.
\item H-LPT continues improving at all three sizes.
\item Small cache growth does not guarantee useful extra computation.
\end{itemize}
\papersource{Paper, Appendix E, PDF pp. 20--23; Table 7, p. 21.}
\end{frame}

\begin{frame}{The contribution is a learned read--write division of labor}
\begin{enumerate}
\item \textbf{Architecture:} keep sequential depth, share first-pass memory, retain a short local window.
\medskip
\item \textbf{Quality:} lower PPL than the main looped baselines; H-LPT gives the strongest memory trade-off.
\medskip
\item \textbf{Mechanism:} distinct key geometry plus a useful direct gradient route to memory construction.
\end{enumerate}
\medskip
\alert{The cache is both a reusable forward representation and a route for learning what to store.}
\papersource{Synthesis of Secs. 3, 5--8, PDF pp. 3--10. Role specialization is an interpretation supported by probes.}
\end{frame}

\begin{frame}{The deployment choice depends on the binding resource}
\begin{itemize}
\item \textbf{Memory-limited concurrency:} sharing admits larger batches; three-pass H-LPT is an attractive measured point.
\medskip
\item \textbf{Single-request latency:} repeated passes remain expensive.
\medskip
\item \textbf{Compute budget:} no FLOPs-matched larger STD was trained; five-pass LPT costs $4.3$--$4.8\times$ STD decoding FLOPs.
\medskip
\item \textbf{Scope:} one corpus, up to 1B parameters, 4K context; long-context tasks remain untested.
\end{itemize}
\papersource{Paper, Sec. 8, PDF p. 10; Appendix F, pp. 24--26. Resource-based interpretation by this presentation.}
\end{frame}

\begin{frame}{Paper reference and discussion}
\textbf{Giovanni Monea, Keshav Ramji, Yousef El-Kurdi, Luis A. Lastras, Yoav Artzi, Nathan Godey, and Ram\'on Fernandez Astudillo.}
\medskip

\emph{The Surprising Effectiveness of Shared Memory in Looped Transformers.}\newline
arXiv:2610.02383v1, 1 October 2026.
\medskip

\href{https://arxiv.org/abs/2610.02383v1}{arxiv.org/abs/2610.02383v1}
\begin{block}{Discussion}
If memory construction can be reused across depth, how much recurrent compute should be spent writing versus reading it?
\end{block}
\papersource{All experimental figures in this deck are original paper figures. Selected tables are transcribed from the paper.}
\end{frame}

\appendix

\begin{frame}{Backup: all baselines answer different questions}
\begin{tabular}{lp{10.8cm}}
\toprule
Baseline & Comparison role\\
\midrule
STD & One pass, same-size parameter reference.\\[3pt]
LT / LT2 & Direct full-attention / hybrid sharing controls.\\[3pt]
PLT & Shared first-pass memory with gated dual attention and parallel recurrence.\\[3pt]
Loopie & Repeat a layer before advancing to the next layer.\\[3pt]
DeepStd & Untied, deeper and narrower; matches effective depth and approximate parameters.\\[3pt]
SPS & Separate memory and prediction residual streams; listed at the two-pass cost point.\\
\bottomrule
\end{tabular}\par
\medskip
PLT differs from LPT in multiple design choices; its gap does not isolate the softmax choice alone.
\papersource{Paper, Sec. 4, PDF p. 4; Sec. 7, pp. 8--9; Appendix E, pp. 20--21.}
\end{frame}

\begin{frame}{Backup: seed evidence is strongest at S}
\centering
\begin{tabular}{lrrr}
\toprule
S-size comparison & $R$ & Mean PPL gap $\pm$ 95\% CI & One-sided $p$\\
\midrule
LPT $-$ LT & 2 & $-0.40\pm0.05$ & $6.5\times10^{-5}$\\
 & 3 & $-0.71\pm0.05$ & $9.4\times10^{-6}$\\
 & 5 & $-1.05\pm0.04$ & $2.4\times10^{-6}$\\
H-LPT $-$ LT2 & 5 & $-0.13\pm0.02$ & $1.6\times10^{-4}$\\
LPT $-$ PLT & 2 & $-0.07\pm0.09$ & $4.8\times10^{-2}$\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Four paired seeds; initialization and data order both change.
\item Intervals use Student-$t$; tests use paired same-seed differences.
\item M/L and downstream accuracy have less replication coverage.
\end{itemize}
\papersource{Paper, Appendix B, PDF pp. 15--16, Tables 3--4. Negative gaps favor the first model.}
\end{frame}

\begin{frame}{Backup: gradient bars are projected contributions}
\begin{itemize}
\item Give each recursion an equal-valued independent parameter copy for the backward probe.
\item Detach shared KV reads to isolate $g_{\mathrm{hidden}}$; then
\end{itemize}
\[
g_{\mathrm{KV}}=g^{(1)}-g_{\mathrm{hidden}}.
\]
\[
p_a=\frac{\langle g_a,g^{(1)}\rangle}{\|g^{(1)}\|_2},\qquad
p_{\mathrm{KV}}+p_{\mathrm{hidden}}=\|g^{(1)}\|_2.
\]
\begin{itemize}
\item Raw path norms are not additive.
\item 32 sequences $\times$ 32 sampled loss positions; final loss in every probe.
\item Excludes embeddings, readout, recurrent norm, and halting gate.
\end{itemize}
\papersource{Paper, Appendix D, PDF pp. 17--19; Eq. (5), p. 18. Error bars resample sequences, not training seeds.}
\end{frame}

\begin{frame}{Backup: a larger KV path already appears at initialization}
\fig{fig13-init}{0.95}{0.61}
\medskip
\begin{itemize}
\item More recursions create more shared-cache reads even before training.
\item Trained ratios are larger; detachment is the stronger learning intervention.
\end{itemize}
\papersource{Paper, Fig. 13, PDF p. 20. Geometric mean norm ratios; fresh random weights, not saved initial checkpoints.}
\end{frame}

\begin{frame}{Backup: implementation details explain the cost accounting}
\begin{itemize}
\item Training: concatenate shared/local KV and use a FlexAttention block mask.
\medskip
\item Decoding: combine two attention outputs using log-sum-exp weights to reproduce a joint softmax.
\medskip
\item GPU timing: preallocated caches, whole-step CUDA graphs, random weights; maximum feasible batch per model.
\medskip
\item Hybrid measured state is 3--17\% above the analytic estimate: fp32 GDN state plus short-convolution buffers.
\end{itemize}
\papersource{Paper, Sec. 3, PDF p. 4; Appendix A, p. 15; Appendix F, p. 24.}
\end{frame}

\begin{frame}{Backup: quality claims should retain their metric and setting}
\begin{itemize}
\item H-LPT beats LT2 on main-table PPL, but two-pass M/L accuracy is lower by 0.6/0.9 points.
\medskip
\item The Introduction labels a 1.09 PPL reduction as ``nats.'' PPL is dimensionless; NLL uses nats/token.
\medskip
\item Under better normalization, LT can learn with final loss and the quality gap narrows.
\medskip
\item Nine-pass 1B LPT degrades relative to five passes; H-LPT still improves in this experiment.
\end{itemize}
\papersource{Paper, Table 1, PDF p. 4; Introduction, p. 2; Table 5, p. 17; Table 7, p. 21.}
\end{frame}

\begin{frame}{Backup: a concrete query separates time from recursion}
\textbf{Teaching example:} one layer, token position $i=5$, $R=3$, $\omega=2$.
\medskip
\centering
\begin{tabular}{lll}
\toprule
Recursion & First-pass entries & Current-pass entries\\
\midrule
$t=1$ & Positions $1,2,3,4,5$ & No extra local set\\[4pt]
$t=2$ & Positions $1,2,3,4,5$ & Positions $3,4,5$ from pass 2\\[4pt]
$t=3$ & Positions $1,2,3,4,5$ & Positions $3,4,5$ from pass 3\\
\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item The current local entry is included; $\omega$ counts the two preceding positions.
\item Two representations of the same token remain distinct entries.
\item Position 6 is never visible. Prediction passes leave first-pass KV intact.
\end{itemize}
\papersource{Teaching expansion of Eq. (3), PDF p. 3. All positions assumed to lie in the same document.}
\end{frame}

\begin{frame}{Backup: two attention kernels can implement one softmax}
For one query, let $a_{s,j}$ and $a_{l,j}$ be its shared/local scores,
including scaling, position information, and masking.
\[
\ell_s=\log\sum_j e^{a_{s,j}},\qquad
o_s=\sum_j e^{a_{s,j}-\ell_s}v_{s,j}
\]
Define $\ell_l,o_l$ analogously and $\ell=\log(e^{\ell_s}+e^{\ell_l})$. Then
\[
\boxed{o=e^{\ell_s-\ell}o_s+e^{\ell_l-\ell}o_l.}
\]
\begin{itemize}
\item The coefficients are the two sides' joint attention masses.
\item They sum to one and depend on query--key scores.
\item Separate cache storage still permits exact joint normalization.
\end{itemize}
\papersource{Teaching derivation of the log-sum-exp merge described in Appendix F, PDF p. 24.}
\end{frame}

\end{document}