% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}


\title[Looped Latent Attention]{Looped Latent Attention}
\subtitle{Cross-Loop KV Compression for Looped Transformers}
\author{Paper: James O'Neill and Fergal Reid, Fin AI Research\\[4pt]\small Independent research reading}
\date{arXiv:2607.15456v2 (24 July 2026)\\Reading: 20 September 2026}
\begin{document}
\begin{frame}
\titlepage
\vspace{-5mm}
\begin{center}\small Compress the recurrent KV trajectory while preserving loop-specific states.\end{center}
\end{frame}

\begin{frame}{Weight sharing leaves a request-sized memory bill}
\begin{block}{The concrete mismatch}
A shared $D$-layer stack runs $T$ times, but each loop produces its own K/V cache.
\end{block}
\[
\text{KV scalars per token}=2TDH d_{\rm head}
\]
\begin{itemize}
\item Ouro-1.4B: $D=24$, $T=4$, $H=16$, $d_{\rm head}=128$.
\item Parameters of a shared stack; KV state across 96 effective layers.
\item Long context and large batches amplify this per-request cost.
\end{itemize}
\papersource{Paper, Secs. 1 and 3, PDF pp. 1--4; Appendix A, p. 14.}
\end{frame}

\begin{frame}{The strongest redundancy lies along recurrence}
\centering
\includegraphics[width=\linewidth,height=.42\textheight,keepaspectratio]{figures/fig-02-structure.png}
\begin{itemize}
\item Loop effective rank: roughly $0.61$--$0.63$ of its maximum.
\item The first direction carries about 60\% of energy; other axes are flatter.
\item V stabilizes more slowly than K: allocate $r_v>r_k$.
\end{itemize}
\papersource{Original Fig. 2, Sec. 4, PDF p. 5. Convergence alone does not imply low rank.}
\end{frame}

\begin{frame}{Store one code; reconstruct the requested loop}
\begin{block}{For one token, layer, head, and K or V}
\[
x=[x_1;\ldots;x_T]\in\mathbb R^{Td_{\rm head}},\qquad
c=(x-\mu)W_{\rm down}\in\mathbb R^r
\]
\[
\hat x_t=cW_{{\rm up},t}^{\top}+\mu_t
\]
\end{block}
\begin{itemize}
\item $W_{\rm down}$: $Td_{\rm head}\times r$; $W_{{\rm up},t}$: $d_{\rm head}\times r$.
\item Separate K and V codes; loop-specific means and reconstruction maps.
\item Fit pre-RoPE content; apply the original RoPE after reconstruction.
\end{itemize}
\papersource{Paper, Eqs. (1)--(3), Sec. 3, PDF pp. 3--4. Row-vector convention.}
\end{frame}

\begin{frame}{The exact saving concerns persistent KV scalars}
\[
\rho=\frac{2THd_{\rm head}}{(r_k+r_v)H}
=\frac{2Td_{\rm head}}{r_k+r_v}
\]
\begin{block}{Worked example: Ouro-1.4B}
$T=4$, $d_{\rm head}=128$, $(r_k,r_v)=(96,160)$:
\[
\rho=\frac{1024}{256}=4,\qquad
768\;\text{KiB/token}\longrightarrow192\;\text{KiB/token}.
\]
\end{block}
\begin{itemize}
\item Same precision; fixed codec weights and scratch buffers are excluded.
\item Every token retains its own code: cache size still grows with context.
\end{itemize}
\papersource{Eq. (4), PDF p. 4; Table 5, p. 11. Numerical substitution is a teaching example.}
\end{frame}

\begin{frame}{Teacher geometry initializes a small conversion problem}
\begin{enumerate}
\item Collect pre-RoPE K/V on 255 calibration prompts and center them.
\item Use top-$r$ right singular vectors of stacked-loop activations.
\item Freeze the teacher; refine only codec parameters.
\end{enumerate}
\[
\mathcal L=\mathrm{KL}(p_{\rm teacher}\Vert p_{\rm swap})
+\frac{0.5}{TD}\sum_{t,d}\|a_{t,d}-\hat a_{t,d}\|_2^2
\]
\begin{itemize}
\item Typical conversion: 3000 steps, batch 8, length 512, AdamW.
\item Select the best held-out KL checkpoint; ablations use 300 steps.
\end{itemize}
\papersource{Eq. (5), PDF p. 4; Appendix A, pp. 14--15. $a_{t,d}$ is the attention output.}
\end{frame}

\begin{frame}{Compare axes at the same persistent-cache budget}
\centering
\begin{tabular}{lll}
\toprule
Method & Compressed axis & Role \\
\midrule
LLA & Loop & Per-head trajectory code \\
LLA-2D & Loop and head & One code per layer/token \\
mla\_head & Head & Separate code for each loop \\
cla & Layer & Adjacent-layer reuse \\
kv\_quant & Precision & Inference-only quantization \\
Final-loop reuse & Loop endpoint & Zero-parameter control \\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item LLA-2D saves another factor $H$ only at \emph{fixed ranks}.
\item Matched-budget comparisons need different ranks across variants.
\item A failed endpoint control is not a direct evaluation of trained MELT.
\end{itemize}
\papersource{Secs. 2--3, PDF pp. 2--4; Appendix A, p. 14.}
\end{frame}

\begin{frame}{Evaluation separates fidelity, generation, and capacity}
\begin{itemize}
\item \textbf{Teacher-forced fidelity:} held-out KL and top-1 agreement.
\item \textbf{Free generation:} GSM8K, MATH-500, code, and BBH.
\item \textbf{Likelihood tasks:} MC suites on given tokens.
\item \textbf{Systems:} fabricated decode caches pushed to OOM on one H200.
\end{itemize}
\begin{block}{Protocol matters}
1.4B iso-cache codecs: 100 GSM8K items; teacher: 1319.\\
Cold-start sweep: 200 codec items, earlier streaming decoder.\\
Most quality runs use two-pass reconstruction, storing full K/V.
\end{block}
\papersource{Sec. 5, PDF p. 6; Appendix A, pp. 14--15; Sec. 7, p. 11.}
\end{frame}

\begin{frame}{At 4$\times$, a trajectory code beats endpoint reuse}
\centering
\begin{tabular}{lrrr}
\toprule
Ouro-1.4B & KL $\downarrow$ & GSM8K $\uparrow$ & MC-avg $\uparrow$\\
\midrule
Teacher ($1\times$) & --- & .794 & .646\\
Final-loop reuse & --- & .000 & .443\\
\textbf{LLA} & \textbf{.059} & \textbf{.800} & .640\\
Head codec & .122 & .522 & .585\\
Layer sharing & .267 & .660 & .597\\
KV quantization & --- & .702 & .645\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Loop states are compressible but functionally distinct.
\item Quantization is competitive at 4$\times$; LLA does not win every metric.
\end{itemize}
\papersource{Selected rows of Table 1, PDF p. 7. Teacher and codec sample counts differ.}
\end{frame}

\begin{frame}{Broad-suite numbers limit the ``near-lossless'' claim}
\centering
\begin{tabular}{lrrrr}
\toprule
Ouro-1.4B & GSM8K & HumanEval & MATH-500 & MMLU-Pro\\
\midrule
Teacher & .794 & .707 & .74 & .481\\
LLA $1.33\times$ & .795 & .756 & .70 & .467\\
LLA $2\times$ & .750 & .744 & .63 & .469\\
LLA $4\times$ & .590 & .610 & .52 & .416\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Light compression is often close; sensitivity remains task-dependent.
\item Table 7 contradicts the Fig. 5 caption's ``code/MMLU-Pro through 4$\times$'' claim.
\item The .590 and .800 GSM8K results belong to different protocols.
\end{itemize}
\papersource{Table 7, PDF p. 16; Fig. 5/Table 1, p. 7; Appendix A, p. 14. HumanEval: base pass@1.}
\end{frame}

\begin{frame}{The loop-axis advantage also appears at 2.6B}
\centering
\begin{tabular}{lrrr}
\toprule
GSM8K EM & $4\times$ & $5.33\times$ & $10.67\times$\\
\midrule
LLA & .834 & .817 & .821\\
Head codec & .585 & .772 & .775\\
KV quantization & .814 & .351 & ---\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Teacher $\approx .83$; LLA and head arms use all 1319 items.
\item MC-avg: teacher .653; LLA .645/.646/.640 across these budgets.
\item This supports transfer of the ordering, not monotonic gains with compression.
\end{itemize}
\papersource{Table 2, PDF p. 8; Table 11, p. 19; Appendix A, p. 14.}
\end{frame}

\begin{frame}{SVD initialization drives the largest ablation gain}
\centering
\begin{tabular}{lr}
\toprule
4$\times$ codec, 300-step conversion & Held-out KL $\downarrow$\\
\midrule
SVD + ranks 96/160 + attention matching & .105\\
First-loop SVD initialization & .128\\
Random initialization & .771\\
Equal ranks 128/128 & .129\\
KL only & .110\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Random initialization has about $7.3\times$ the reference KL.
\item More V capacity helps; attention-output matching adds a smaller gain.
\item No multi-seed significance estimate is supplied for these differences.
\end{itemize}
\papersource{Table 10, Fig. 12, PDF p. 18.}
\end{frame}

\begin{frame}{Huginn transfers the idea to 32 recurrent steps}
\centering
\begin{tabular}{lrrrr}
\toprule
Compression & SVD KL & SVD top-1 & Distilled KL & MC-avg\\
\midrule
$16\times$ & .005 & .970 & --- & .523\\
$32\times$ & .010 & .954 & --- & .525\\
$64\times$ & .197 & .799 & .145 & .486\\
$128\times$ & .763 & .620 & .177 & .473\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Teacher MC-avg: .523; SVD alone is strong through 32$\times$.
\item Distillation recovers fidelity at higher compression.
\item These are decoder-independent results, not long-generation or serving tests.
\end{itemize}
\papersource{Table 3, Sec. 5, PDF p. 9. Top-1 means token agreement with the teacher.}
\end{frame}

\begin{frame}{On-policy distillation visits the deployed model's prefixes}
\[
\mathcal L_{\rm off}=\mathbb E_{y\sim p_T}
\left[\frac1{|y|}\sum_t\mathrm{KL}\big(p_T(\cdot\mid y_{<t})\Vert p_S^\theta(\cdot\mid y_{<t})\big)\right]
\]
\[
\mathcal L_{\rm on}=\mathbb E_{\tilde y\sim\mathrm{sg}(p_S^\theta)}
\left[\frac1{|\tilde y|}\sum_t\mathrm{KL}\big(p_T(\cdot\mid\tilde y_{<t})\Vert p_S^\theta(\cdot\mid\tilde y_{<t})\big)\right]
\]
\begin{itemize}
\item Same frozen teacher target; change the prefix distribution.
\item Stop gradients through sampling; update only the codec.
\item No reward or correctness filtering is used.
\end{itemize}
\papersource{Eqs. (6)--(7), Sec. 6, PDF pp. 9--10. $T$: teacher; $S$: student.}
\end{frame}

\begin{frame}{Fewer missing answers need not mean higher accuracy}
\centering
\begin{tabular}{lrrr}
\toprule
MATH-500 & Off-policy & On-policy & No-answer: off $\to$ on\\
\midrule
$3.2\times$ & .46 & .70 & .21 $\to$ .06\\
$4\times$ & .43 & .66 & .15 $\to$ .03\\
$5.33\times$ & .53 & .69 & .12 $\to$ .03\\
$10.67\times$ & .65 & .69 & .09 $\to$ .07\\
$21.33\times$ & .66 & .62 & .11 $\to$ .05\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item At 4$\times$: +23 percentage points in final-answer accuracy.
\item At 21.33$\times$: fewer missing answers, but lower accuracy.
\item Rollout correction cannot guarantee recovery of lost information.
\end{itemize}
\papersource{Table 4, PDF p. 10. Teacher accuracy .74; main codec evaluations use 200 items.}
\end{frame}

\begin{frame}{Compression reduces the usable retrieval horizon}
\centering
\includegraphics[width=.94\linewidth,height=.51\textheight,keepaspectratio]{figures/fig-08-passkey.pdf}
\begin{itemize}
\item $10.67\times$: perfect to 16k; .80 at 32k and .73 at 64k.
\item Exception to the prose: $4\times$ at 64k is .87, below the .9 threshold.
\item 15 trials per cell; 2.6B's 64k OOM is a memory failure.
\end{itemize}
\papersource{Original Fig. 8, PDF p. 11; Table 12, p. 20. Latent-store decoder.}
\end{frame}

\begin{frame}{Capacity improves while current throughput falls}
\centering
\begin{tabular}{lrrr}
\toprule
Ouro-1.4B, one H200 & Teacher & LLA $4\times$ & LLA $21.3\times$\\
\midrule
Persistent KV / token & 768 KiB & 192 KiB & 36 KiB\\
Max batch at 4096 tokens & 32 & 128 & 768\\
Full-KV fast-path ms/token & 63.6 & 74.1 & ---\\
Capacity-path peak tokens/s & 748 & --- & 108\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item Full-KV fast path keeps reconstructed states: no memory saving.
\item Latent-store path saves memory but reconstructs on every read.
\item Batch capacity is measured with fabricated decode caches to OOM.
\end{itemize}
\papersource{Table 5, Sec. 7, PDF p. 11. These rows describe distinct execution paths.}
\end{frame}

\begin{frame}{Absorbed attention exposes a speed--fidelity gap}
\[
q^\top(W_{\rm up}c)=(W_{\rm up}^{\top}q)^\top c
\]
\begin{itemize}
\item Moving projections outside attention can avoid explicit reconstruction.
\item Full RoPE blocks direct absorption; a decoupled branch changes fidelity.
\end{itemize}
\begin{columns}[T]
\begin{column}{.49\textwidth}
\begin{block}{Kernel geometry benchmark}
Random weights; 262k context.\\
Absorbed: 4.25 ms/token.\\
Teacher: 4.46 ms/token.\\
$2.30\times$ faster than reconstruction.
\end{block}
\end{column}
\begin{column}{.49\textwidth}
\begin{block}{Post-hoc fidelity: KL}
Full RoPE: .059.\\
Decoupled: .55.\\
With query adapter: .34.\\
Extended training: .30.
\end{block}
\end{column}
\end{columns}
\papersource{Appendix J, Tables 13--14, PDF pp. 20--21. Column-vector identity; speed is a microbenchmark.}
\end{frame}

\begin{frame}{Three evidence boundaries govern the interpretation}
\begin{enumerate}
\item \textbf{Preserve protocols.} Checkpoints, decoders, and evaluation subsets differ across tables. Broad-suite GSM8K repeats cold-start numbers despite different protocol descriptions.
\medskip
\item \textbf{Check figures against tables.} The 4$\times$ code/MMLU-Pro claim and the blanket 64k retrieval claim are too strong.
\medskip
\item \textbf{Separate system goals.} Exact KV storage savings, sampled-task quality, and random-weight kernel speed are distinct results.
\end{enumerate}
\papersource{Reading assessment based on Secs. 5--7 and Appendices A, C, I, J. No independent rerun.}
\end{frame}

\begin{frame}{Loop trajectories are a useful compression target}
\begin{block}{What the paper establishes}
A learned low-rank cross-loop codec can preserve useful behavior while substantially reducing persistent KV state in the tested recurrent models.
\end{block}
\begin{itemize}
\item SVD exploits teacher geometry; loop-specific reconstruction matters.
\item On-policy conversion helps long rollouts at several budgets.
\item Capacity gains are real; faithful end-to-end speed remains unresolved.
\end{itemize}
\medskip
\small James O'Neill and Fergal Reid. \emph{Looped Latent Attention: Cross-Loop KV Compression for Looped Transformers}.\\
\href{https://arxiv.org/abs/2607.15456v2}{arXiv:2607.15456v2}, 24 July 2026.\\
\papersource{Read: full PDF, main\_minimal.tex, results\_space\_optimized.tex, figures, and references.}
\end{frame}

\appendix
\begin{frame}{Backup: cold-start ranks define a separate frontier}
\centering
\begin{tabular}{lrrr}
\toprule
Compression & $(r_k,r_v)$ & GSM8K & Held-out train-KL\\
\midrule
$4.00\times$ & (96,160) & .590 & .0160\\
$3.20\times$ & (120,200) & .575 & .0153\\
$2.67\times$ & (144,240) & .575 & .0157\\
$2.00\times$ & (192,320) & .750 & .0155\\
$1.60\times$ & (240,400) & .755 & .0070\\
$1.33\times$ & (288,480) & .795 & .0035\\
\bottomrule
\end{tabular}\par
\medskip
Teacher reference .820 on 100 items; codecs on 200 items.\\
Earlier streaming reconstruction decoder; do not merge with Table 1.
\papersource{Table 8, PDF p. 17; Appendix A, p. 14.}
\end{frame}

\begin{frame}{Backup: the full memory path needs two passes}
\begin{enumerate}
\item Capture teacher pre-RoPE K/V across loops.
\item Encode and reconstruct the captured vectors.
\item Re-forward with reconstructed K/V injected before RoPE.
\end{enumerate}
\begin{block}{What is retained?}
Quality path: full post-RoPE reconstructed K/V.\\
Capacity path: latents, with reconstruction on read.\\
Long prefill: stream layer-wise reconstruction and free captures.
\end{block}
The mathematical codec does not establish a cost-free single-pass decoder.
\papersource{Appendix A, PDF p. 14; Appendix J, p. 20.}
\end{frame}

\begin{frame}{Backup: token eviction composes in storage}
\centering
\begin{tabular}{lr}
\toprule
260 retained tokens: sink 4 + window 256 & GSM8K\\
\midrule
Eviction on a near-lossless 8-bit codec & .510\\
LLA $4\times$ + eviction & .520\\
\bottomrule
\end{tabular}\par
\medskip
\begin{itemize}
\item About 74\% of the few-shot prompt is evicted; quality cost is substantial.
\item At $21.3\times$, 1024 kept tokens give $36\,\mathrm{KiB}\times1024=36\,\mathrm{MiB}$.
\item A 1M-token input with eviction is not lossless storage of 1M tokens.
\end{itemize}
\papersource{Table 9, PDF p. 17; Appendix J, p. 20.}
\end{frame}

\begin{frame}{Backup: length bins reveal a rollout failure mode}
\centering
\includegraphics[width=.88\linewidth,height=.62\textheight,keepaspectratio]{figures/fig-07-exposure.png}\par
\small On-policy tracks the teacher more closely at 3.2$\times$. Generated length also reflects task difficulty; this plot alone is not a controlled causal test.
\papersource{Original Fig. 7, PDF p. 10; figure labels $n=300$ per arm.}
\end{frame}
\end{document}