% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[LaCT]{Test-Time Training Done Right}
\subtitle{Large Chunk Test-Time Training (LaCT)}
\author{Paper by Tianyuan Zhang et al.\\MIT and Adobe Research}
\date{arXiv:2505.23884v1 (29 May 2025)\\Paper discussion prepared 21 September 2026}
\begin{document}
\begin{frame}\titlepage\end{frame}
\begin{frame}{The contribution is a capacity--efficiency tradeoff}
\begin{itemize}
\item Write context into fast weights in large chunks: 2K tokens to about 1M, depending on the task.
\item Use the resulting GPU efficiency to scale nonlinear memory and richer update rules.
\item Keep local window attention for fine-grained structure and causality.
\item Evidence: novel view synthesis, language modeling, and autoregressive video diffusion.
\end{itemize}
\papersource{Paper, Secs. 1--2, pp. 1--3. Three separate task-specific models.}
\end{frame}
\begin{frame}{Fast weights are a temporary neural memory}
\[
\text{Write: }W\leftarrow W-\eta\nabla_W\mathcal L(f_W(k),v)
\qquad \text{Read: }o=f_W(q).
\]\begin{itemize}
\item Slow weights learn across training samples; fast weights adapt within a context.
\item The same memory mechanism runs during training and inference.
\item A fixed state compresses history, so storage does not grow with every token.
\item Compression limits what can be retrieved exactly.
\end{itemize}
\papersource{Paper, Sec. 2.1, Eqs. (1)--(2), p. 2.}
\end{frame}
\begin{frame}{Small updates can leave matrix units underused}
\[
r_{\rm intensity}=\frac{2h^2b}{2h^2+4hb}
=\frac{b}{1+2b/h}\leq\min(b,h/2).
\]\begin{itemize}
\item For BF16 matmul: FLOPs divided by bytes moved.
\item Both chunk size $b$ and matrix width $h$ constrain arithmetic intensity.
\item Large chunks offer sequence-dimension parallelism within each update.
\item Reported peak utilization is a layer-level result, not universal end-to-end utilization.
\end{itemize}
\papersource{Paper, Sec. 2.2, Eq. (3), Fig. 1, p. 3.}
\end{frame}
\begin{frame}{Two paths handle two scales of dependency}
\begin{itemize}
\item Window attention: local token structure and fine-grained causality.
\item Large-chunk TTT: compressed history in a nonlinear weight state.
\item A conventional feed-forward network supplies channel mixing.
\item Language/video share QKV between TTT and window branches; outputs are combined.
\end{itemize}
\papersource{Paper, Fig. 2, p. 4; Algorithm 2, p. 20.}
\end{frame}
\begin{frame}{The memory is a SwiGLU network}
\[
f_W(x)=W_2[\operatorname{SiLU}(W_1x)\circ(W_3x)]
\]
\[
\mathcal L_i=-f_W(k_i)^\top v_i,\qquad
 g=\nabla_W\sum_{i=1}^{b}\eta_i\mathcal L_i.
\]\begin{itemize}
\item $x,k_i,v_i\in\mathbb R^h$; $W_1,W_3\in\mathbb R^{rh\times h}$ and $W_2\in\mathbb R^{h\times rh}$.
\item $h$: head width; $r$: hidden-width multiplier. The objective is negative dot product, not MSE.
\item Token-dependent positive $\eta_i$ controls write importance; normalization stabilizes the state.
\end{itemize}
\papersource{Paper, Sec. 3.1, Eqs. (4)--(7), p. 4; Algorithm 1, p. 19.}
\end{frame}
\begin{frame}{Muon changes how the write uses matrix directions}
\[
 g=U\Sigma V^\top,\quad \operatorname{Muon}(g)\approx UV^\top,
\qquad W\leftarrow\operatorname{L2Norm}(W-\operatorname{Muon}(g)).
\]\begin{itemize}
\item Five Newton--Schulz iterations approximate gradient orthogonalization.
\item With momentum: $M\leftarrow\bar\beta M+g$, then transform $M$.
\item Large chunks amortize matrix update overhead.
\item Muon helps key NVS/language results; video experiments do not use it.
\end{itemize}
\papersource{Paper, Sec. 3.2, p. 5; Appendix A, pp. 18--20; C.3, p. 27.}
\end{frame}
\begin{frame}{Read/write order defines the dependency mask}
\begin{center}\includegraphics[width=0.98\linewidth,height=0.52\textheight,keepaspectratio]{figures/mask.pdf}\end{center}\begin{itemize}
\item Update then apply: current chunk can communicate internally.
\item Apply then update: read only previous chunks; no future leakage.
\end{itemize}
\papersource{Original Fig. 3, p. 5. Masks describe dependencies, not numerical attention equivalence.}
\end{frame}
\begin{frame}{Language needs both delayed writes and a causal window}
\begin{block}{Teaching example: chunk length 4}
Tokens 5--8 read the state written by tokens 1--4. A causal local window handles recent tokens. Write tokens 5--8 only after reading the chunk.
\end{block}\begin{itemize}
\item Window length at least chunk length covers the missing recent dependencies.
\item Actual chunks/windows: 2048 or 4096 tokens.
\item Low-rank learnable initialization reduces slow parameters, not the full online state.
\end{itemize}
\papersource{Paper, Sec. 4.2, p. 7; Fig. 10 and Appendix B.2, p. 22; C.2, pp. 25--26.}
\end{frame}
\begin{frame}{For image sets, prefill builds a scene-specific memory}
\begin{itemize}
\item Input: posed RGB images; query: camera rays of target views.
\item Within-image attention preserves local spatial structure.
\item Each layer writes all input views, then reads for input and target tokens.
\item After prefill, render new views with fixed fast weights; targets do not write back.
\end{itemize}
\papersource{Paper, Sec. 4.1, pp. 6--7; Fig. 9 and Appendix B.1, pp. 20--21.}
\end{frame}
\begin{frame}{Video writes clean context, not denoising candidates}
\[
S=[X_1^{\rm noise},X_1,X_2^{\rm noise},X_2,\ldots],\qquad
X_i^{\rm noise}=(1-t_i)X_i+t_i\epsilon.
\]\begin{itemize}
\item One chunk = 3 latent frames = 4680 visual tokens.
\item Only clean chunks update persistent memory; noisy chunks read it.
\item A local window spans two chunks with a causal inter-chunk mask.
\item 1.3B, 5-second training repeats noisy chunks twice; 14B does not.
\end{itemize}
\papersource{Paper, Sec. 4.3, p. 7; Fig. 11, p. 23; Appendix C.3, pp. 27--28.}
\end{frame}
\begin{frame}{Large chunks also support distributed writes}
\[
g=\sum_j\nabla_W\sum_{i\in\mathrm{shard}\ j}\eta_i\mathcal L_i.
\]\begin{itemize}
\item All devices compute local gradients at the same old state.
\item All-reduce the gradients before nonlinear Muon and normalization.
\item NVS: split tokens inside a chunk; reported throughput overhead 1--3\%.
\item Video: shard TTT heads, switching between sequence and head layouts.
\end{itemize}
\papersource{Paper, Sec. 3.4, p. 6; Appendices E--F, pp. 29--32.}
\end{frame}
\begin{frame}{The three evaluations test different capabilities}
\begin{tabular}{p{2.3cm}p{5.5cm}p{4.2cm}}
\toprule Task & Setup & Main metric \\\midrule
View synthesis & Objaverse/GSO; DL3DV; 312M model & PSNR, SSIM, LPIPS \\
Language & 760M/40B and 3B/60B training tokens; 32K context & Per-position loss; retrieval \\
Video & Wan 2.1, 1.3B and 14B; proprietary training data & Held-out denoising loss \\\bottomrule
\end{tabular}\begin{itemize}
\item Linear-cost baselines receive matching local attention.
\item Maximum lengths and training protocols must not be mixed across tasks.
\end{itemize}
\papersource{Paper, Sec. 5 and Table 1, pp. 7--11; Appendix C, pp. 24--28.}
\end{frame}
\begin{frame}{NVS substantially reduces prefill time}
\begin{center}\begin{tabular}{lrrr}
\toprule Model & Parameters & Prefill (s)$\downarrow$ & Render FPS$\uparrow$\\\midrule
Full attention & 284M & 16.1 & 2.3\\
Perceiver attention & 287M & 16.8 & 34.4\\
LaCT & 312M & 1.4 & 38.7\\\bottomrule
\end{tabular}\end{center}\begin{itemize}
\item A100; 48 input images at $512\times512$; 196K input tokens.
\item Calculated: $11.5\times$ faster prefill and $16.8\times$ rendering throughput versus full attention.
\item These are separate phases, not a single end-to-end speedup.
\end{itemize}
\papersource{Paper, Table 2, p. 8. Ratios calculated from reported values.}
\end{frame}
\begin{frame}{View quality stays close to full attention}
\begin{center}\includegraphics[width=0.98\linewidth,height=0.52\textheight,keepaspectratio]{figures/nvs.pdf}\end{center}\begin{itemize}
\item GSO, $512^2$, 48 views: 39.1 dB LaCT vs. 39.3 dB full attention.
\item DL3DV: 24.7 to 28.9 dB as input views rise from 16 to 128.
\item The million-token result uses posed images, not text retrieval.
\end{itemize}
\papersource{Original Fig. 4, p. 8; Tables 5--6, p. 25.}
\end{frame}
\begin{frame}{Language curves show useful long-range memory}
\begin{center}\includegraphics[width=0.98\linewidth,height=0.52\textheight,keepaspectratio]{figures/lm.pdf}\end{center}\begin{itemize}
\item Lower later-position loss than GLA/DeltaNet in the reported settings.
\item Selected retrieval curves favor LaCT Muon.
\item The complete appendix tables reveal important failure cases.
\end{itemize}
\papersource{Original Fig. 5, p. 9; full retrieval results in Tables 8--9, p. 27.}
\end{frame}
\begin{frame}{Complex retrieval remains far below full attention}
\begin{center}\begin{tabular}{lrr}
\toprule 3B model, 32K context & Single-needle avg. & Multi-needle avg.\\\midrule
Transformer & 91.2 & 43.0\\
GLA + SWA & 13.3 & 11.7\\
DeltaNet + SWA & 28.4 & 14.8\\
LaCT Momentum & 40.6 & 14.2\\
LaCT Muon & 48.0 & 21.2\\\bottomrule
\end{tabular}\end{center}\begin{itemize}
\item Accuracy in percent; higher is better.
\item Single-needle gain vs. DeltaNet: +19.6 percentage points.
\item Gap to Transformer: 43.2 points. Muon is not best in every table cell.
\end{itemize}
\papersource{Paper, Tables 8--9, p. 27. Averages use the three reported tasks in each group.}
\end{frame}
\begin{frame}{Capacity improves without becoming the fastest model}
\begin{center}\begin{tabular}{lrr}
\toprule Model & Tokens/s/GPU & State elements/block\\\midrule
Transformer & 4.1K & ---\\
Transformer SWA & 6.4K & ---\\
GLA SWA & 5.0K & $384d$\\
DeltaNet SWA & 5.1K & $128d$\\
LaCT Momentum & 4.9K & $2304d$\\
LaCT Muon & 4.3K & $2304d$\\\bottomrule
\end{tabular}\end{center}\begin{itemize}
\item 3B, 32K, $d=3072$; 8 A100-40GB GPUs, FSDP and checkpointing.
\item LaCT has $6\times$ GLA / $18\times$ DeltaNet fast-weight state.
\end{itemize}
\papersource{Paper, Table 3, p. 9; Appendix C.2 and Table 7, p. 26.}
\end{frame}
\begin{frame}{Video loss supports memory utility within trained lengths}
\begin{center}\includegraphics[width=0.98\linewidth,height=0.52\textheight,keepaspectratio]{figures/video.pdf}\end{center}\begin{itemize}
\item Comparable to full block-causal attention; better than SWA/Mamba+SWA.
\item Longer-video evaluation follows longer-video fine-tuning.
\item 14B: 56,160 visual tokens; about 107K interleaved training tokens.
\end{itemize}
\papersource{Original Fig. 6, p. 10; Appendix C.3, pp. 27--28. No zero-shot length extrapolation claim.}
\end{frame}
\begin{frame}{The ablations support the combined design}
\begin{center}\includegraphics[width=0.98\linewidth,height=0.52\textheight,keepaspectratio]{figures/ablation.pdf}\end{center}\begin{itemize}
\item Nonlinear memory beats larger linear memory in these comparisons.
\item Equal-state linear large chunks help NVS, but underperform GLA in language.
\item The winning language variant also adds larger state and Muon.
\end{itemize}
\papersource{Original Fig. 8, p. 11; state/optimizer analysis in Fig. 7, p. 10.}
\end{frame}
\begin{frame}{Interpret state size using actual matrix dimensions}
\[
S=3n_hh^2r=\frac{3rd^2}{n_h},\qquad
\mathrm{FLOPs}_{\rm write+read}\approx6nS.
\]\begin{itemize}
\item $n_h$: head count; $d=n_hh$; $n$: tokens; $r$: hidden-width multiplier. Three matrices and nine main matmuls.
\item NVS default: $6d^2$ per block; about 84M fast weights in 312M parameters.
\item The 40\% state/parameter ratio is a larger-state configuration.
\item Momentum, attention cache, activations and Muon cost are additional.
\end{itemize}
\papersource{Derivation from Eq. (6), p. 4 and Algorithm 1, p. 19; Appendix A, p. 18 has notation errors.}
\end{frame}
\begin{frame}{The evidence has clear boundaries}
\begin{itemize}
\item NVS requires camera poses; unposed reconstruction is untested.
\item Language reasoning and much larger language models are untested.
\item Fixed-size memory still loses retrieval detail; no unlimited-capacity guarantee.
\item Video uses proprietary data and denoising loss rather than a complete perceptual evaluation.
\item Q/K rotation invariance is absent in the proposed fast-weight branch.
\end{itemize}
\papersource{Paper, Sec. 7, p. 13; reader analysis based on Tables 8--9, p. 27.}
\end{frame}
\begin{frame}{Take away the design principle, with its conditions}
\begin{block}{Main idea}
Amortize memory writes over large chunks, spend the resulting efficiency on expressive state, and preserve local structure with attention.
\end{block}\begin{itemize}
\item Strong evidence for an NVS quality--speed tradeoff.
\item Useful language/video results, with task-dependent limitations.
\item Open question: how does the tradeoff change at matched total compute and peak memory?
\end{itemize}
\papersource{Paper, Sec. 8, p. 13. Final question is a discussion prompt, not a paper result.}
\end{frame}
\begin{frame}{Source and reading scope}
\textbf{Tianyuan Zhang, Sai Bi, Yicong Hong, Kai Zhang, Fujun Luan, Songlin Yang, Kalyan Sunkavalli, William T. Freeman, Hao Tan.}\par\medskip
\emph{Test-Time Training Done Right}. arXiv:2505.23884v1, 2025.\par
\url{https://arxiv.org/abs/2505.23884v1}\par\medskip
Read: full paper and Appendices A--F; same-version author TeX and PDF.\par\medskip
Figures are cropped from the original PDF. Tables select reported values. Interpretations and derived ratios are identified. No experimental reproduction or project-video assessment was performed.
\papersource{Prepared as a research-group paper discussion, approximately 25--30 minutes.}
\end{frame}
\end{document}