\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs,array}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\newcommand{\src}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\LLA}{\mathrm{LLA}}
\newcommand{\NW}{\mathrm{NW}}
\newcolumntype{P}[1]{>{\raggedright\arraybackslash}p{#1}}

\begin{document}

\begin{frame}[plain]
\vspace{0.5cm}
{\Huge\color{AcademicBlue}\bfseries Local Linear Attention\par}
\vspace{0.35cm}
{\Large An Optimal Interpolation of Linear and Softmax Attention\par}
{\Large for Test-Time Regression\par}
\vspace{0.45cm}
{\normalsize Yifei Zuo, Yutong Yin, Zhichen Zeng, Ang Li,\par Banghua Zhu, Zhaoran Wang\par}
\vspace{0.35cm}
{\small \href{https://arxiv.org/abs/2510.01450v1}{arXiv:2510.01450v1} \quad
\href{https://proceedings.iclr.cc/paper_files/paper/2026/hash/f9af970430ac401594ce01022fccb049-Abstract-Conference.html}{Published at ICLR 2026} \quad
\href{https://github.com/Yifei-Zuo/FlashLLA}{Official code}\par}
\vfill
{\footnotesize Paper discussion \hfill 3 November 2026\par}
\end{frame}

\begin{frame}{A query-specific linear fit improves local prediction}
\begin{block}{One-sentence takeaway}
LLA fits a local slope and intercept around each query; FlashLLA makes the resulting solve memory-feasible.
\end{block}
\begin{itemize}
\item Statistics: local linear regression addresses boundary bias of local averaging.
\item Systems: pairwise differences and per-query covariance matrices need not be stored.
\item Evidence: strong synthetic non-stationary regression and recall; state tracking remains weak.
\end{itemize}
\src{Paper, Abstract and Secs. 2--5, PDF pp. 1, 3--11.}
\end{frame}

\begin{frame}{Attention as a regression solver at test time}
\[
\mathcal D_i=\{(k_j,v_j):j\le i\},\qquad
\hat f_i\leftarrow \mathrm{Fit}(\mathcal D_i),\qquad
o_i=\hat f_i(q_i).
\]
\begin{center}
\begin{tabular}{P{0.27\linewidth}P{0.34\linewidth}P{0.29\linewidth}}
\toprule
Mechanism & Local hypothesis & Main consequence \\
\midrule
Linear / MesaNet & one global linear fit & compact state; model bias \\
Softmax & constant near each query & local, but boundary bias \\
LLA & slope + intercept near each query & local and first-order correction \\
\bottomrule
\end{tabular}
\end{center}
\src{Paper, Sec. 2.1 and Fig. 1, PDF pp. 2--4.}
\end{frame}

\begin{frame}{The three estimators make different errors}
\begin{columns}[T]
\begin{column}{0.44\textwidth}
\centering\includegraphics[width=\linewidth,height=0.65\textheight,keepaspectratio]{figures/front-lla-annotate.png}
\end{column}
\begin{column}{0.54\textwidth}
\centering\includegraphics[width=\linewidth,height=0.65\textheight,keepaspectratio]{figures/front-lla.png}
\end{column}
\end{columns}
\begin{center}Global misspecification $\rightarrow$ local averaging $\rightarrow$ local slope correction.\end{center}
\src{Original paper, Fig. 1, PDF p. 2. Illustrative regression diagrams.}
\end{frame}

\begin{frame}{Global linear regression can retain non-vanishing bias}
\begin{itemize}
\item If the true mapping $f$ is not globally affine, every global affine fit has an approximation-error floor.
\item A local-constant kernel fit can shrink its neighborhood with sample size.
\end{itemize}
\[
\mathbb E\!\int_D\!\|\hat f_{\mathrm{GL}}-f\|^2=\Omega(1),\qquad
\mathbb E\!\int_D\!\|\hat f_{\NW}-f\|^2=O\!\left(n^{-3/(d+3)}\right).
\]
\begin{block}{Interpretation}
Locality can eliminate global model misspecification, although local averaging still has a boundary problem.
\end{block}
\src{Paper, Proposition 2.1, PDF p. 4; proof in Appendix A, pp. 15--19. Conditional asymptotic result.}
\end{frame}

\begin{frame}{A local slope removes first-order boundary bias}
\begin{center}
\begin{tabular}{lcc}
\toprule
Estimator & Integrated squared bias & Variance \\
\midrule
Local constant & $O(h^3)$ & $O((nh^d)^{-1})$ \\
Local linear & $O(h^4)$ & $O((nh^d)^{-1})$ \\
\bottomrule
\end{tabular}
\end{center}
\vspace{0.2cm}
\[
\mathrm{IMSE}_{\NW}=\Omega\!\left(n^{-3/(d+3)}\right),\qquad
\mathrm{IMSE}_{\mathrm{LL}}=O\!\left(n^{-4/(d+4)}\right).
\]
\begin{itemize}
\item The boundary layer has width $O(h)$; its local-constant bias is $O(h)$.
\item Integrated MSE over $D$; requires a sufficiently large boundary-normal gradient and optimal bandwidths.
\end{itemize}
\src{Paper, Proposition 2.2, PDF p. 5; Appendix B, pp. 20--24.}
\end{frame}

\begin{frame}{LLA predicts with the intercept at the query}
\[
z_{ij}=k_j-q_i,\qquad
\min_{b,W}\ \frac12\sum_{j\le i}w_{ij}\|v_j-b-Wz_{ij}\|_2^2
   +\lambda\|W\|_F^2.
\]
\begin{columns}[T]
\begin{column}{0.49\textwidth}
\begin{block}{Locality}
$w_{ij}$ emphasizes keys near the current query $q_i$.
\end{block}
\end{column}
\begin{column}{0.49\textwidth}
\begin{block}{First-order structure}
$W$ captures a local slope; $b=\hat f(q_i)$ is the output.
\end{block}
\end{column}
\end{columns}
\src{Paper, Sec. 2.3, Eq. (6), PDF p. 5.}
\end{frame}

\begin{frame}{The optimal intercept is a corrected weighted sum}
\[
\begin{aligned}
\omega_i&=\sum_j w_{ij}, & \mu_i&=\sum_jw_{ij}z_{ij},\\
\Sigma_i&=\sum_jw_{ij}z_{ij}z_{ij}^{\top}+\lambda I,
&\rho_i&=\Sigma_i^{-1}\mu_i.
\end{aligned}
\]
\[
\boxed{\quad o_i=\sum_{j\le i}s_{ij}v_j,\qquad
s_{ij}=\frac{w_{ij}(1-z_{ij}^{\top}\rho_i)}{\omega_i-\mu_i^{\top}\rho_i}\quad}
\]
\begin{itemize}
\item $\sum_j s_{ij}=1$, but individual $s_{ij}$ may be negative.
\item As $\lambda\rightarrow\infty$, $\rho_i\rightarrow 0$: recover local-constant weights $w_{ij}/\omega_i$.
\end{itemize}
\src{Paper, Eqs. (7)--(8), PDF p. 5.}
\end{frame}

\begin{frame}{The interpolation is a residual correction}
\[
\boxed{\quad o_i=\hat W_iq_i+
\sum_{j\le i}\frac{w_{ij}}{\omega_i}(v_j-\hat W_i k_j)\quad}
\]
\begin{enumerate}
\item Predict from the fitted linear map $\hat W_i$.
\item Locally average its residuals to correct misspecification.
\end{enumerate}
\begin{block}{Meaning of ``optimal interpolation''}
The local weighted regression objective determines $\hat W_i$ and the correction; this is not a fixed convex mixture of two attention outputs.
\end{block}
\src{Paper, Sec. 2.3, Eq. (9), PDF p. 5.}
\end{frame}

\begin{frame}{Two algebraic changes remove the memory bottlenecks}
\begin{center}
\begin{tabular}{P{0.28\linewidth}P{0.31\linewidth}P{0.31\linewidth}}
\toprule
Naive object & Memory & Replacement \\
\midrule
All $k_j-q_i$ & $\Theta(n^2d)$ & separate key and query terms \\
All $\Sigma_i$ & $\Theta(nd^2)$ & matrix-free CG products \\
\bottomrule
\end{tabular}
\end{center}
\vspace{0.25cm}
\[
\tilde\mu_i=\sum_jw_{ij}k_j,\qquad
\mu_i=\tilde\mu_i-\omega_iq_i.
\]
\[
\Sigma_i p=\sum_jw_{ij}(k_j-q_i)\bigl((k_j-q_i)^\top p\bigr)+\lambda p.
\]
\src{Paper, Sec. 3.1, Eqs. (10)--(12), PDF p. 6.}
\end{frame}

\begin{frame}{FlashLLA streams key blocks in three stages}
\begin{enumerate}
\item \textbf{Statistics:} scan key blocks; update $\omega_i$ and $\mu_i$ with online-max stabilization.
\item \textbf{Solve:} run CG for $\Sigma_i\rho_i=\mu_i$ without storing $\Sigma_i$.
\item \textbf{Output:} rescan key/value blocks and accumulate $\sum_js_{ij}v_j$.
\end{enumerate}
\begin{block}{Trade-off}
Working memory becomes $\Theta(nd)$, while each CG iteration streams keys again. Compute and I/O remain substantial.
\end{block}
\src{Paper, Algorithm 1, PDF p. 7; CG details in Appendix C, pp. 24--25.}
\end{frame}

\begin{frame}{The measured gain is peak memory, not runtime}
\begin{columns}[c]
\begin{column}{0.67\textwidth}
\centering\includegraphics[width=\linewidth,height=0.72\textheight,keepaspectratio]{figures/memory.pdf}
\end{column}
\begin{column}{0.31\textwidth}
\begin{itemize}
\item One H200 GPU
\item Head dimension $d=128$
\item Several batch sizes
\item Naive method reaches memory limits quickly
\end{itemize}
\end{column}
\end{columns}
\src{Original paper, Fig. 2, PDF p. 7; profiling description, p. 8.}
\end{frame}

\begin{frame}{Four tasks separate adaptation, recall, and state updates}
\begin{center}
\begin{tabular}{P{0.27\linewidth}P{0.39\linewidth}P{0.23\linewidth}}
\toprule
Task & Question & Metric \\
\midrule
Test-time regression & Adapt without trained projections? & position MSE $\downarrow$ \\
In-context regression & Learn from changing linear segments? & test MSE $\downarrow$ \\
MQAR recall & Retrieve discrete key--value pairs? & accuracy $\uparrow$ \\
State tracking & Apply sequences of swaps? & accuracy $\uparrow$ \\
\bottomrule
\end{tabular}
\end{center}
\src{Paper, Sec. 4, PDF pp. 8--10; protocols in Appendix E, pp. 27--29.}
\end{frame}

\begin{frame}{LLA adapts after piecewise-linear shifts}
\centering\includegraphics[width=0.98\linewidth,height=0.68\textheight,keepaspectratio]{figures/single_row_fit_dim64_seqlen1024.pdf}
\begin{itemize}
\item $L=1024$, $d=64$; 10,000 sampled sequences; lower Loss/Dim is better.
\item MesaNet can win before the first shift; LLA stays low after shifts.
\end{itemize}
\src{Original paper, Fig. 3, PDF p. 8; configuration in Appendix E.2, p. 28.}
\end{frame}

\begin{frame}{The benefit grows with dimension, with exceptions}
\begin{columns}[c]
\begin{column}{0.61\textwidth}
\centering\includegraphics[width=\linewidth,height=0.72\textheight,keepaspectratio]{figures/scaling.pdf}
\end{column}
\begin{column}{0.37\textwidth}
\begin{itemize}
\item Ratio $=$ comparator total MSE / LLA total MSE.
\item Above 1 favors LLA.
\item MesaNet beats LLA for some low-dimensional, long-segment settings.
\end{itemize}
\end{column}
\end{columns}
\src{Original paper, Fig. 4 and Sec. 4, PDF p. 8. Log axes.}
\end{frame}

\begin{frame}{Trained LLA leads on non-stationary in-context regression}
\centering\includegraphics[width=0.86\linewidth,height=0.59\textheight,keepaspectratio]{figures/icrg-subplots.pdf}
\begin{itemize}
\item Two-layer models without MLPs; $d_x=d_y=32$, model width 128, two heads.
\item At $L=256,S=64$: test loss $\approx0.07$ for LLA versus $\approx0.52$ for Softmax (read from plot).
\end{itemize}
\src{Original paper, Fig. 5(a), PDF p. 9; three-seed best-hyperparameter scores.}
\end{frame}

\begin{frame}{LLA's largest recall gains appear on hard long contexts}
\begin{columns}[c]
\begin{column}{0.67\textwidth}
\centering\includegraphics[width=\linewidth,height=0.73\textheight,keepaspectratio]{figures/zoo.pdf}
\end{column}
\begin{column}{0.31\textwidth}
\begin{itemize}
\item MQAR vocabulary: 8,192 tokens.
\item At 512--32, LLA $\approx43\%$ vs Softmax $\approx11\%$.
\item Gated DeltaNet matches or exceeds LLA in some easier cases.
\end{itemize}
\end{column}
\end{columns}
\src{Original paper, Fig. 5(b), PDF p. 9; approximate plot readings.}
\end{frame}

\begin{frame}{LLA does not solve permutation state tracking}
\begin{columns}[c]
\begin{column}{0.65\textwidth}
\centering\includegraphics[width=\linewidth,height=0.72\textheight,keepaspectratio]{figures/state_tracking.pdf}
\end{column}
\begin{column}{0.33\textwidth}
\begin{itemize}
\item LLA tracks Softmax closely.
\item At $N=48$: $\approx52\%$ vs Gated DeltaNet $\approx98\%$.
\item Sequential swaps favor a state update mechanism.
\end{itemize}
\end{column}
\end{columns}
\src{Original paper, Fig. 6, PDF p. 10. First x-axis label is 12; Appendix E.2 lists 16.}
\end{frame}

\begin{frame}{The evidence supports a focused conclusion}
\begin{itemize}
\item The rate comparison assumes smooth $f$, a smooth domain, a compact radial kernel, and tuned bandwidths; the experiments use piecewise data and exponentiated dot-product weights.
\item FlashLLA demonstrates memory scaling, but CG adds computation and repeated I/O.
\item Large-language-model training and real-language benchmark gains remain untested in this paper.
\end{itemize}
\begin{block}{Practical question}
Does a cheaper approximate slope preserve the regression/recall advantage at equal end-to-end compute?
\end{block}
\src{Paper, Sec. 5, PDF pp. 10--11; assumptions in Appendices A--B, pp. 15--24.}
\end{frame}

\begin{frame}{Take-home messages}
\begin{enumerate}
\item \textbf{Statistical design:} fit a query-specific local line rather than a local constant.
\item \textbf{Implementation:} algebraic separation plus matrix-free CG avoids quadratic and $nd^2$ intermediate storage.
\item \textbf{Observed trade-off:} strong adaptation and recall on tested synthetic tasks; weak sequential state tracking and higher compute cost.
\end{enumerate}
\vspace{0.45cm}
\begin{center}\Large $\displaystyle\text{locality}+\text{first-order fit}\ \longrightarrow\ \text{better local prediction}$\end{center}
\src{Synthesis of the paper's Secs. 2--5, PDF pp. 3--11.}
\end{frame}

\begin{frame}{Sources}
\begin{itemize}
\item Y. Zuo et al., \emph{Local Linear Attention: An Optimal Interpolation of Linear and Softmax Attention For Test-Time Regression}, \href{https://arxiv.org/abs/2510.01450v1}{arXiv:2510.01450v1} (2025).
\item \href{https://proceedings.iclr.cc/paper_files/paper/2026/hash/f9af970430ac401594ce01022fccb049-Abstract-Conference.html}{Official ICLR 2026 proceedings record}.
\item \href{https://github.com/Yifei-Zuo/FlashLLA}{Author's FlashLLA repository} (accessed 3 November 2026).
\end{itemize}
\vspace{0.4cm}
All plotted figures in this deck are from the arXiv v1 author source and are identified by figure number and PDF page on the corresponding slides.
\end{frame}

\appendix
\begin{frame}{Backup: where the rate exponents come from}
\[
\begin{aligned}
\mathrm{NW:}\quad & h^3+(nh^d)^{-1}
&&\Rightarrow h\asymp n^{-1/(d+3)},\quad \mathrm{MSE}\asymp n^{-3/(d+3)},\\[0.25cm]
\mathrm{LL:}\quad & h^4+(nh^d)^{-1}
&&\Rightarrow h\asymp n^{-1/(d+4)},\quad \mathrm{MSE}\asymp n^{-4/(d+4)}.
\end{aligned}
\]
\begin{itemize}
\item The local-constant boundary layer has width $O(h)$ and pointwise squared bias $O(h^2)$.
\item A local line removes the first-order term, leaving $O(h^4)$ squared bias.
\end{itemize}
\src{Paper, Appendices A--B, PDF pp. 15--24.}
\end{frame}

\begin{frame}{Backup: experimental configurations that matter}
\begin{center}
\begin{tabular}{P{0.27\linewidth}P{0.58\linewidth}}
\toprule
Task & Configuration \\
\midrule
Test-time regression & $L=1024$; $d=8,16,32,64,128$; 10,000 sequences \\
In-context regression & $L=64,128,256,512$; 100k train and 1k test per seed; 16 queries; train noise 0.1, test noise 0 \\
MQAR & $L=64,128,256,512$; KV count $L/16,L/8,L/4$; short convolution disabled \\
State tracking & 8 queries; swaps $S\sim\mathrm{Uniform}(N/6,N/3)$ \\
\bottomrule
\end{tabular}
\end{center}
\src{Paper, Appendix E.2, PDF pp. 28--29.}
\end{frame}

\end{document}