% Compile with pdflatex. Replace the demonstration content before delivery. \documentclass[aspectratio=169,11pt]{beamer} \usepackage[T1]{fontenc} \usepackage[utf8]{inputenc} \usepackage{amsmath,amssymb,graphicx,booktabs} \definecolor{AcademicBlue}{RGB}{26,57,86} \definecolor{AcademicTeal}{RGB}{0,105,112} \setbeamercolor{structure}{fg=AcademicBlue} \setbeamercolor{frametitle}{fg=AcademicBlue,bg=white} \setbeamercolor{title}{fg=AcademicBlue} \setbeamercolor{alerted text}{fg=AcademicTeal} \setbeamercolor{block title}{fg=white,bg=AcademicBlue} \setbeamercolor{block body}{fg=black,bg=AcademicBlue!5} \setbeamertemplate{navigation symbols}{} \setbeamertemplate{itemize items}[circle] \setbeamertemplate{footline}{% \hfill\usebeamerfont{page number in head/foot}% \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}} \setbeamersize{text margin left=8mm,text margin right=8mm} \setbeamerfont{frametitle}{size=\Large,series=\bfseries} \setbeamerfont{title}{size=\LARGE,series=\bfseries} \newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}} \addtobeamertemplate{frametitle}{\vspace{2mm}}{} \setlength{\tabcolsep}{9pt} \title[Kaczmarz Linear Attention]{Kaczmarz Linear Attention} \subtitle{A normalized step for residual memory writes} \author[Paper reading presentation]{Paper: Jiaxuan Zou, Ruifeng Ren, Yong Liu\\[2mm]\small Research group reading presentation} \date{arXiv:2605.08587v1 \quad $\vert$ \quad 13 November 2026} \hypersetup{pdfauthor={Research group reading presentation on Zou, Ren and Liu (2026)}} \newcommand{\Sbar}{\widetilde S} \newcommand{\norm}[1]{\lVert #1\rVert_2} \newcommand{\diag}{\operatorname{Diag}} \begin{document} \begin{frame} \titlepage \end{frame} \begin{frame}{KLA normalizes the residual write by key energy} \begin{block}{One-line change to the paper's GDN template} \[ \beta_t=\eta_t \quad\longrightarrow\quad \boxed{\beta_t=\frac{\eta_t}{\norm{k_t}^2+\varepsilon}} \] \end{block} \begin{itemize} \item Preserve the state, decay gate, residual direction and chunk solver. \item Derive the ideal step from a minimum-change projection. \item Test language modeling, controlled memory tasks and runtime. \end{itemize} \medskip {\small \href{https://arxiv.org/abs/2605.08587v1}{Paper: arXiv v1, 9 May 2026}.\\ Publication not found as of 2026-11-13; \href{https://jiaxuanzou0714.github.io/publications/}{author page}: Under Review.\\ \href{https://github.com/anonymous-kla-review/kla-anonymous}{Official code: public anonymized implementation}.} \papersource{Paper, Abstract and Secs. 1, 3; PDF pp. 1--5. Status checked 2026-11-13.} \end{frame} \begin{frame}{Residual writes edit an existing association} \[ S_t\in\mathbb R^{d_k\times d_v},\quad k_t,q_t\in\mathbb R^{d_k},\quad v_t\in\mathbb R^{d_v} \] \begin{align*} \Sbar_{t-1}&=\alpha_t S_{t-1} &&\text{decay old memory}\\ e_t&=v_t-\Sbar_{t-1}^{\top}k_t &&\text{predict the missing value}\\ S_t&=\Sbar_{t-1}+\beta_t k_te_t^{\top} &&\text{edit the active key direction} \end{align*} \begin{itemize} \item Additive attention writes $k_tv_t^\top$ without subtracting the prediction. \item GDN uses a learned scalar $\beta_t=\eta_t$ for the residual write. \item KLA keeps the direction and changes its length. \end{itemize} \papersource{Paper, Sec. 2, PDF pp. 2--3. Scalar-gated notation used throughout.} \end{frame} \begin{frame}{Key norm changes the effective correction} \[ e_t^+=v_t-S_t^\top k_t =\bigl(1-\beta_t\norm{k_t}^2\bigr)e_t \] \begin{block}{The effective correction is $\beta_t\norm{k_t}^2$} Large key energy can cause overshoot; small energy can under-correct. \end{block} \medskip \textbf{Teaching example:} scalar state, $\Sbar=0$, $k=2$, $v=1$. \begin{center} \begin{tabular}{lccc} \toprule Step & $\beta$ & New state $S$ & Readout $Sk$\\ \midrule Unnormalized & $1$ & $2$ & $4$\\ Kaczmarz & $1/4$ & $1/2$ & $1$\\ \bottomrule \end{tabular} \end{center} \papersource{Paper, Sec. 3.1, Eq. (4), PDF p. 4. Example: presentation derivation.} \end{frame} \begin{frame}{Exact projection makes the smallest feasible state change} \[ \min_S\frac12\lVert S-\Sbar_{t-1}\rVert_F^2 \qquad\text{subject to}\qquad S^\top k_t=v_t \] \begin{enumerate} \item For $k_t\ne0$, the optimal correction has form $k_tc^\top$. \item The constraint gives $c\norm{k_t}^2=e_t$. \item Hence the minimum-change update is \end{enumerate} \[ \boxed{S_t=\Sbar_{t-1}+\frac{k_te_t^\top}{\norm{k_t}^2}} \] \medskip It changes only the current key direction and exactly fits this key-value pair. \papersource{Paper, Sec. 3.1, Eqs. (1)--(2), PDF p. 3; Appendix A, pp. 13--14.} \end{frame} \begin{frame}{Three views yield the same ideal coefficient} \[ \mathcal L_t(S)=\frac12\norm{S^\top k_t-v_t}^2,\qquad \nabla_S\mathcal L_t(\Sbar_{t-1})=-k_te_t^\top \] \[ \mathcal L_t(\Sbar_{t-1}+\tau k_te_t^\top) =\frac12(1-\tau\norm{k_t}^2)^2\norm{e_t}^2 \] \begin{itemize} \item Exact line search yields $\tau^*=1/\norm{k_t}^2$ for nonzero residual. \item The correction is orthogonal to every $H$ with $H^\top k_t=0$. \item The same step is normalized gradient descent on the state loss. \end{itemize} \medskip \alert{Exact projection: $k_t\ne0$, $\eta_t=1$, $\varepsilon=0$.} \papersource{Paper, Theorem 1 and Eq. (4), PDF p. 4; Appendix A.1, pp. 13--14.} \end{frame} \begin{frame}{Relaxation controls the remaining residual} \[ \beta_t=\frac{\eta_t}{\norm{k_t}^2+\varepsilon},\qquad \eta_t=\sigma(\text{input projection}),\quad\varepsilon=10^{-6} \] \[ \boxed{e_t^+=\left(1-\eta_t\frac{\norm{k_t}^2}{\norm{k_t}^2+\varepsilon}\right)e_t} \] \begin{itemize} \item For nonzero key and $0<\eta_t\le1$, the factor is in $[0,1)$. \item With $\varepsilon=0$, the gate directly selects the corrected fraction. \item Comparison is against the \emph{decayed} state $\Sbar_{t-1}$. \item At zero key with positive $\varepsilon$, the write vanishes. \end{itemize} \papersource{Paper, Eq. (3), Proposition 2 / Eq. (5), PDF pp. 3--5; Appendix C, p. 18.} \end{frame} \begin{frame}{The inference interface stays the same} \begin{enumerate} \item Decay: $\Sbar_{t-1}=\alpha_t S_{t-1}$. \item Predict residual: $e_t=v_t-\Sbar_{t-1}^\top k_t$. \item Compute the normalized scalar $\beta_t$. \item Write: $S_t=\Sbar_{t-1}+\beta_tk_te_t^\top$. \item Read: $o_t=S_t^\top q_t/\norm{q_t}$. \end{enumerate} \begin{block}{Implementation condition that matters} If the actual write key is already unit-normalized, the denominator is $1+\varepsilon$ and the change nearly disappears. The paper explicitly specifies query normalization; baseline key processing needs clarification. \end{block} \papersource{Paper, Sec. 3.3 and Algorithm 1, PDF p. 5. Unit-key observation: algebraic deduction.} \end{frame} \begin{frame}{Training preserves the GDN triangular solve} \[ \gamma_i=\alpha_1\cdots\alpha_i,\quad B=\diag(\beta_i),\quad D_\gamma=\diag(\gamma_i) \] \[ A_{ij}=\gamma_i/\gamma_j\ (j\le i),\qquad A^- = \text{strict lower part of }A \] \begin{block}{Effective residual writes $U\in\mathbb R^{C\times d_v}$} \[ \left[I+B(A^-\odot KK^\top)\right]U=B(V-D_\gamma KS_0) \] \end{block} \begin{itemize} \item $K,Q\in\mathbb R^{C\times d_k}$ and $V\in\mathbb R^{C\times d_v}$. \item Only the values in $B$ change relative to the GDN solver. \item Fixed head dimensions and chunk length give linear sequence scaling. \end{itemize} \papersource{Paper, Sec. 4, Eq. (7), PDF pp. 5--6. Nonzero cumulative decay assumed.} \end{frame} \begin{frame}{The main experiment fixes architecture and token budget} \begin{center} \begin{tabular}{ll} \toprule Model & 0.4B parameters; 24 layers; hidden size 1024\\ Heads / FFN & 8 heads, head dimension 128; FFN expansion $4\times$\\ Data / context & 1B SlimPajama tokens; 4K training context\\ Optimizer & AdamW, $(0.9,0.95)$; weight decay 0.1\\ Schedule & Cosine; 2\% warmup; gradient clipping 1.0\\ Resources & 4 A100 80GB GPUs; about 4 h per run\\ \bottomrule \end{tabular} \end{center} \medskip \begin{itemize} \item Pretraining: KLA, GDN, DeltaNet, GLA, Longhorn and Mamba2. \item Other comparisons: KLA, GDN and Mamba2. \item No multi-seed pretraining; synthetic tasks use seed 42. \end{itemize} \papersource{Paper, Sec. 5.1, PDF p. 6; Appendix C, p. 18. Configurations as reported.} \end{frame} \begin{frame}{KLA reduces pretraining PPL from 8.50 to 8.09} \begin{center} \begin{tabular}{lrrrrrr} \toprule Model & KLA & DeltaNet & GLA & GDN & Longhorn & Mamba2\\ \midrule PPL $\downarrow$ & \alert{8.09} & 8.26 & 8.42 & 8.50 & 8.79 & 9.41\\ \bottomrule \end{tabular} \end{center} \bigskip \begin{block}{0.4B parameters, 1B tokens, 4K training context} A decrease of 0.41 PPL relative to GDN, or about 4.82\%.\\ The gap to the next-best baseline, DeltaNet, is 0.17 PPL. \end{block} \medskip This is the main fixed-budget language-modeling result. \papersource{Paper, Table 1, PDF p. 2; Sec. 5.2 and Fig. 1, p. 6. Relative change calculated here.} \end{frame} \begin{frame}{KLA has lower PPL at every tested context length} \begin{center} \begin{tabular}{lrrrrrrr} \toprule Context & 1K & 2K & 4K & 8K & 16K & 32K & 65K\\ \midrule KLA & 39.1 & 39.8 & 40.0 & 43.6 & 45.3 & 44.5 & 43.6\\ GDN & 42.9 & 43.5 & 43.8 & 47.4 & 49.4 & 48.4 & 47.5\\ Mamba2 & 49.9 & 50.9 & 52.2 & 56.4 & 58.7 & 57.3 & 56.7\\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item PPL $\downarrow$; no long-context fine-tuning. \item GDN gap: 3.7--4.1 PPL at each tested length. \item At 65K: 43.6 versus 47.5, about 8.21\% lower. \end{itemize} \begin{block}{Keep this protocol separate from Table 1} At 4K, Table 3 reports 40.0 / 43.8; Table 1 reports 8.09 / 8.50. The paper does not sufficiently reconcile their evaluation pipelines. \end{block} \papersource{Paper, Table 3 and Sec. 5.2, PDF p. 7. All table values are PPL.} \end{frame} \begin{frame}{Synthetic tasks probe distinct memory demands} \begin{center} \begin{tabular}{lll} \toprule Task & Memory demand & Protocol\\ \midrule MQAR & Multiple key-value lookups & Train 256; test to 2048\\ S-NIAH & One association amid distractors & Test 1K--8K\\ Palindrome & Ordered reverse reproduction & Test 256--2048\\ Stack & Repeated LIFO push/pop edits & Test 256--2048\\ \bottomrule \end{tabular} \end{center} \medskip \begin{itemize} \item MQAR: 2-layer model, 10K steps. \item S-NIAH: 6K steps; Palindrome / Stack: 20K steps. \item S-NIAH specifies exact match; the other tasks report accuracy. \end{itemize} \papersource{Paper, Sec. 5.3, PDF pp. 7--8; Appendix C, p. 18. Synthetic seed: 42.} \end{frame} \begin{frame}{MQAR gains emerge at the longest extrapolation} \begin{center} \begin{tabular}{lrrrr} \toprule Length & 256 & 512 & 1024 & 2048 ($8\times$)\\ \midrule KLA & 99.00 & 98.56 & 95.64 & \alert{73.84}\\ GDN & 98.74 & 98.36 & 94.31 & 66.81\\ Mamba2 & \textbf{99.08} & \textbf{99.02} & \textbf{96.54} & 64.29\\ \bottomrule \end{tabular} \end{center} \bigskip \begin{block}{At $8\times$ the training length} KLA improves over GDN by \alert{7.03 percentage points}.\\ Relative accuracy improvement: approximately 10.52\%. \end{block} \medskip Mamba2 leads at shorter lengths; the ranking changes at 2048. \papersource{Paper, Table 4, PDF p. 8. MQAR validation accuracy (\%); train length 256.} \end{frame} \begin{frame}{Retrieval improves; ordered reversal remains a boundary} \begin{center} \begin{tabular}{llrrr} \toprule Task & Length & KLA & GDN & Mamba2\\ \midrule S-NIAH & 8K & \alert{100.00} & 98.50 & 0.05\\ Palindrome & 2048 & 79.96 & \textbf{80.74} & 0.86\\ Stack & 2048 & \alert{96.94} & 95.64 & 93.82\\ \bottomrule \end{tabular} \end{center} \medskip \begin{itemize} \item S-NIAH: KLA reaches 100\% at all four tested lengths, 1K--8K. \item Stack: a 1.30-point gain at length 2048. \item Palindrome: KLA trails GDN by 0.78 points at length 2048. \end{itemize} \medskip Association editing and ordered storage impose different demands. \papersource{Paper, Table 4 and Fig. 2, PDF p. 8. All values are task accuracies (\%).} \end{frame} \begin{frame}{Prefill matches GDN; measured decode throughput rises} \begin{center} \begin{tabular}{lrrr} \toprule Metric & KLA & GDN & Mamba2\\ \midrule 131K prefill, ms $\downarrow$ & 1220 & 1227 & 1936\\ 32K decode, token/s $\uparrow$ & 3.54 & 1.65 & 2.10\\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item Batch size 1; FLA evaluation path. \item Prefill difference versus GDN: about 0.57\%. \item Decode ratio: $3.54/1.65\approx2.15\times$ (paper: $2.1\times$). \end{itemize} \begin{block}{Interpretation} Reusing the solver explains similar dominant prefill cost. The paper does not isolate why a scalar substitution doubles measured decode throughput. \end{block} \papersource{Paper, Sec. 5.4, PDF p. 7; Fig. 3, p. 9.} \end{frame} \begin{frame}{Normalization and gating have a joint trade-off} \begin{center} \begin{tabular}{lrrr} \toprule Variant & MQAR test $\uparrow$ & MQAR $8\times$ $\uparrow$ & 100M PPL $\downarrow$\\ \midrule KLA default & \textbf{98.39} & 73.84 & \textbf{753.65}\\ No normalization & 0.03 & 0.01 & ---\\ Key-norm only & 97.81 & \textbf{77.68} & 3809.71\\ Learned Scalar & 97.77 & 74.46 & 22968.87\\ \bottomrule \end{tabular} \end{center} \begin{itemize} \item Key-norm only improves extrapolation but worsens short-budget PPL. \item Learned Scalar: trainable parameter initialized at 1.0. \end{itemize} \begin{block}{Configuration ambiguity} NoNorm is described as sigmoid gating, yet it collapses while the GDN reference succeeds. Additional key/module differences are not resolved. \end{block} \papersource{Paper, Table 5, PDF p. 9; Appendix D, pp. 18--19. Accuracy columns in \%.} \end{frame} \begin{frame}{Gating choices trade recall against short-budget PPL} \begin{center} \begin{tabular}{lrrr} \toprule Variant & MQAR test $\uparrow$ & MQAR $8\times$ $\uparrow$ & 100M PPL $\downarrow$\\ \midrule KLA default & 98.39 & \alert{73.84} & 753.65\\ Single Gate & 0.01 & 0.03 & 2104.61\\ GDN reference & \textbf{98.72} & 66.81 & 747.24\\ GLA reference & 0.03 & 0.03 & \textbf{356.01}\\ \bottomrule \end{tabular} \end{center} \bigskip \begin{itemize} \item GLA has the lowest PPL here, but near-zero MQAR accuracy. \item GDN also slightly beats KLA in 100M-token PPL. \item The 1B-token main result and this 100M-token ablation are separate. \end{itemize} \papersource{Paper, Table 5, PDF p. 9; Appendix D, pp. 18--19. Accuracy columns in \%.} \end{frame} \begin{frame}{State expansion trades recall against language modeling} \begin{center} \begin{tabular}{lrrr} \toprule Value expansion & MQAR test $\uparrow$ & MQAR $8\times$ $\uparrow$ & 100M PPL $\downarrow$\\ \midrule $2$ & 98.09 & 71.99 & \textbf{762.33}\\ $4$ & 98.88 & 91.34 & 1669.35\\ $8$ & 95.31 & \textbf{95.37} & 6945.63\\ $16$ & \textbf{99.18} & 92.63 & 1383.38\\ \bottomrule \end{tabular} \end{center} \bigskip \begin{itemize} \item State expansion changes capacity and projection cost. \item It can strongly improve extrapolation on the synthetic task. \item It does not replace the write-strength trade-off in language modeling. \end{itemize} \papersource{Paper, Table 5, PDF p. 9; Appendix D, pp. 18--19. Accuracy columns in \%.} \end{frame} \begin{frame}{Normalized writes still share a fixed-capacity state} For another key $k'$, the same write changes its readout by \[ \Delta(S^\top k')=\beta_t e_t(k_t^\top k'). \] \begin{itemize} \item Orthogonal key directions avoid this write interference. \item Similar keys still interact; normalization does not add memory capacity. \item The local guarantee compares states after decay and before the next token. \item Very small keys make the stabilizer and numerical precision relevant. \end{itemize} \begin{block}{Concrete task boundary} At length 2048, Palindrome accuracy is 79.96\% for KLA and 80.74\% for GDN. \end{block} \papersource{Paper, Sec. 6 and Table 4, PDF pp. 8--9. Cross-key equation: reading derivation.} \end{frame} \begin{frame}{Key energy sets the scale; the gate sets write strength} \begin{enumerate} \item \textbf{Mechanism:} key energy supplies the natural local step scale; $\eta_t$ controls adaptive relaxation. \item \textbf{Algorithm:} the coefficient fits the existing recurrent state and chunkwise triangular solve. \item \textbf{Evidence:} lower 1B-token PPL and stronger long MQAR retrieval; Palindrome and short-budget ablations show concrete trade-offs. \end{enumerate} \medskip \begin{block}{Highest-value follow-up checks} Resolve actual key normalization, NoNorm/GDN configuration differences, and cached decode timing. Then test larger models and multiple seeds. \end{block} \papersource{Paper, Secs. 3--7, PDF pp. 3--10. Follow-up priorities: reading analysis.} \end{frame} \begin{frame}{References and implementation} \begin{itemize} \item Zou, Ren and Liu. \emph{Kaczmarz Linear Attention}.\\ \href{https://arxiv.org/abs/2605.08587v1}{arXiv:2605.08587v1}, 2026. \item \href{https://github.com/anonymous-kla-review/kla-anonymous}{Official anonymized implementation}: model components and experiment scripts. \item \href{https://jiaxuanzou0714.github.io/publications/}{First author's publication page}: Under Review.\\ Publication not found as of 2026-11-13. \item Direct predecessor: Yang, Kautz and Hatamizadeh.\\ \emph{Gated Delta Networks}, ICLR 2025 (paper Ref. 45). \item \href{https://github.com/fla-org/flash-linear-attention}{Flash Linear Attention library} (paper Ref. 42). \end{itemize} \papersource{This presentation reads the specified arXiv v1; paper authors are listed on the title slide.} \end{frame} \appendix \begin{frame}{Backup: the relaxed write solves a proximal problem} \[ \min_S\frac12\lVert S-\Sbar_{t-1}\rVert_F^2+ \frac{\mu_t}{2}\norm{S^\top k_t-v_t}^2 \] \[ \mu_t=\frac{\eta_t}{(1-\eta_t)\norm{k_t}^2+\varepsilon} \] \begin{itemize} \item Stationarity gives $S-\Sbar+\mu_t k_t(S^\top k_t-v_t)^\top=0$. \item Along $S=\Sbar+\tau k_te_t^\top$, \end{itemize} \[ \tau=\frac{\mu_t}{1+\mu_t\norm{k_t}^2} =\frac{\eta_t}{\norm{k_t}^2+\varepsilon}. \] At $\eta_t=1,\varepsilon=0$, use the limit $\mu_t\to\infty$; the displayed finite-penalty formula is singular at that endpoint. \papersource{Paper, Proposition 1, PDF p. 3; Appendix A.3, p. 14. Endpoint clarification: analysis.} \end{frame} \begin{frame}{Backup: a direct expansion gives the chunk equations} \[ S_i=\gamma_i S_0+\sum_{j\le i}\frac{\gamma_i}{\gamma_j}k_ju_j^\top \] \[ u_i=\beta_i\left[v_i-\gamma_i S_0^\top k_i- \sum_{j