% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\usepackage{lmodern}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[Preconditioned DeltaNet]{Preconditioned DeltaNet}
\subtitle{Curvature-aware Sequence Modeling\\for Linear Recurrences}
\author{Paper: Neehal Tumma, Noel Loo, Daniela Rus\\[2mm]\small Independent research paper discussion}
\date{arXiv:2604.21100v1\quad |\quad 22 September 2026}
\begin{document}
\begin{frame}\titlepage\end{frame}
\begin{frame}{A small change to writing improves associative memory}
\begin{block}{Main idea}
Use historical key statistics to scale the \emph{write key}, while keeping the \emph{read key} unchanged.
\end{block}\begin{itemize}
\item Exact inverse-Gram preconditioning connects linear attention and the delta rule.
\item A bounded diagonal approximation retains efficient chunkwise computation.
\item Paired language-model averages improve; individual tasks can regress.
\end{itemize}
\papersource{Secs. 3--4, PDF pp. 3--8. Discussion: approximately 25 minutes.}
\end{frame}

\begin{frame}{DeltaNet already corrects errors, but uses one key twice}
\[
S_t=S_{t-1}+\beta_t\underbrace{(v_t-S_{t-1}k_t)}_{\text{prediction residual}}\quad\underbrace{k_t^\top}_{\text{write direction}}
\]\begin{itemize}
\item $S_t\in\mathbb R^{d_v\times d_k}$ maps keys to values; $o_t=S_tq_t$.
\item The residual reads memory using $k_t$; the update writes along the same $k_t$.
\item One scalar gain does not adapt the update to coordinate-wise key history.
\end{itemize}
\papersource{Secs. 1--2, PDF pp. 1--3.}
\end{frame}

\begin{frame}{The regression view identifies the missing curvature}
\[
\min_S\;\frac12\sum_{i\le t}\|Sk_i-v_i\|_2^2+\frac\lambda2\|S\|_F^2
\]
\[
G_t=\lambda I+\sum_{i\le t}k_ik_i^\top,\quad
C_t=\sum_{i\le t}v_ik_i^\top,\quad S_t^\star=C_tG_t^{-1}
\]\begin{itemize}
\item $G_t$: key Gram; $C_t$: value--key cross-statistic. Take $\lambda>0$ for invertibility.
\item Test-time regression updates an in-sequence memory state, not the outer model parameters.
\item DeltaNet is a per-token gradient step; it does not explicitly invert this curvature.
\end{itemize}
\papersource{Sec. 2.3, Eqs. (1)--(2), PDF p. 2.}
\end{frame}

\begin{frame}{Exact ATQ and ATK produce the same ridge-regression map}
\begin{block}{Apply to query (ATQ)}
$C_t=C_{t-1}+v_tk_t^\top,\quad P_t=G_t^{-1},\quad o_t=C_t(P_tq_t)$.
\end{block}
\begin{block}{Apply to write key (ATK)}
\[
\tilde k_t=\frac{P_{t-1}k_t}{1+k_t^\top P_{t-1}k_t},\qquad
S_t=S_{t-1}+(v_t-S_{t-1}k_t)\tilde k_t^\top
\]
\end{block}
With $S_0=0$ and $P_0=\lambda^{-1}I$, $S_t=C_tP_t$ for every $t$.
Both forms are causal. This exact form has no extra arbitrary gain.
\papersource{Theorem 3.1, Eqs. (3)--(5), PDF p. 3; proof: App. C.1, p. 18.}
\end{frame}

\begin{frame}{A diagonal approximation trades correlations for efficiency}
\[
A_t=A_{t-1}+k_t\odot k_t,\qquad
P_t\approx\operatorname{Diag}(A_t+\lambda\mathbf1)^{-1}
\]\begin{itemize}
\item Only $O(d_k)$ extra statistics, instead of a full $d_k\times d_k$ Gram.
\item Coordinate-wise prefix sums and reciprocals support chunkwise computation.
\item Off-diagonal correlations are discarded. Exact ATQ/ATK equivalence is lost.
\item In general, $(\operatorname{diag}G)^{-1}\ne\operatorname{diag}(G^{-1})$.
\end{itemize}
\papersource{Sec. 3.3, PDF pp. 4--5; App. C.2, p. 19. Ridge shown explicitly here.}
\end{frame}

\begin{frame}{Observed key geometry motivates the diagonal approximation}
\begin{center}\includegraphics[width=.94\linewidth,height=.60\textheight,keepaspectratio]{figures/fig-01.pdf}\end{center}Left: sorted spectra. Right: eigenvalue-weighted coordinate alignment along the sequence. Evidence comes from a selected pretrained DeltaNet-340M layer.
\papersource{Fig. 1, PDF p. 4. This is empirical motivation, not a general approximation guarantee.}
\end{frame}

\begin{frame}{The practical preconditioner is learned, bounded, and monotone}
\begin{align*}
A_t&=\alpha_t^P A_{t-1}+\beta_t^P(k_t\odot k_t),\\
r_t&=\log A_t-\mu,\qquad s_t=\frac{r_t}{\mathbf1+|r_t|},\\
B_t&=\exp[-\log(x)s_t],\qquad \tilde k_t=B_t\odot k_t.
\end{align*}\begin{itemize}
\item Elementwise operations; $\mu>0$ is learned per head; $x=1.5$.
\item $B_{t,j}\in[2/3,3/2]$: larger accumulated activity means smaller write scaling.
\item Uses current statistics and drops the exact-solve normalization.
\item This is not an exact least-squares solver.
\end{itemize}
\papersource{Eq. (9), Sec. 3.5, PDF pp. 5--6; App. E.1/E.3, pp. 25, 27--30.}
\end{frame}

\begin{frame}{One update covers PDN, PGDN, and PKDA}
\[
\bar S_t=S_{t-1}D_t,\qquad
S_t=\bar S_t+\beta_t(v_t-\bar S_tk_t)\tilde k_t^\top,\quad o_t=S_tq_t
\]
\begin{center}\begin{tabular}{lll}\toprule
Model & Main decay $D_t$ & Write key \\\midrule
PDN & $I$ & $B_t\odot k_t$ \\
PGDN & $\alpha_tI$ & $B_t\odot k_t$ \\
PKDA & $\operatorname{Diag}(\boldsymbol\alpha_t)$ & $B_t\odot k_t$ \\\bottomrule
\end{tabular}\end{center}
The statistics have independent gates $\alpha_t^P,\beta_t^P$.
Do not replace the key inside the residual with $\tilde k_t$.
\papersource{Secs. 3.4--3.5, PDF pp. 5--6. Diagonal decay expanded using the stated state dimensions.}
\end{frame}

\begin{frame}{Two keys still admit the compact delta-rule algorithm}
For one chunk, $Q,K,\tilde K\in\mathbb R^{C\times d_k}$ and $V\in\mathbb R^{C\times d_v}$.
\begin{align*}
T&=[I+\operatorname{tril}(\operatorname{Diag}(\beta)K\tilde K^\top,-1)]^{-1}\operatorname{Diag}(\beta),\\
W&=TK,\qquad U=TV,\qquad R=U-WS_{\rm in}^\top,\\
S_{\rm out}&=S_{\rm in}+R^\top\tilde K,\\
O&=QS_{\rm in}^\top+[(Q\tilde K^\top)\odot M]R.
\end{align*}
$M$: causal mask. The triangular solve uses read--write interactions.
State writing and local output use the write key.
\papersource{Eq. (6), PDF p. 4; App. F.2.1, pp. 35--36. Shown without decay.}
\end{frame}

\begin{frame}{Preserving two interaction matrices matters for throughput}
\begin{itemize}
\item Original delta rule: interactions $KK^\top$ and $QK^\top$.
\item Preconditioned delta rule: $K\tilde K^\top$ and $Q\tilde K^\top$.
\item General DPLR needs four distinct interaction paths in the cited implementation.
\item Compute diagonal statistics first; then use the specialized two-key kernel.
\item Backward separates read/write gradients and propagates through the preconditioner.
\end{itemize}
\papersource{App. D, PDF pp. 21--24, Figs. 8--9 and Table 3; App. F, pp. 33--37.}
\end{frame}

\begin{frame}{The eigenvalue bound is useful, but local}
For PGDN with $\|k_t\|_2=1$, $\alpha_t,\beta_t\in[0,1]$:
\[
T_t=\alpha_t(I-\beta_tk_t\tilde k_t^\top),\qquad
\lambda(T_t)=\begin{cases}\alpha_t & (d_k-1\text{ copies}),\\
\alpha_t(1-\beta_tk_t^\top\tilde k_t)&(1\text{ copy}).\end{cases}
\]
Since $k_t^\top\tilde k_t\in[1/x,x]$, $\beta_tx\le2$ ensures $|\lambda(T_t)|\le1$.
\begin{block}{Reading boundary}
A one-step spectral bound does not by itself establish stability of every time-varying matrix product or training gradient. The scalar-decay formula is not directly a PKDA proof.
\end{block}
\papersource{App. E.3, PDF pp. 29--30. Boundary statement: our mathematical interpretation.}
\end{frame}

\begin{frame}{Evaluate within matched architecture pairs}
\begin{center}\begin{tabular}{lll}\toprule
 & Smaller models & Larger models \\\midrule
Training tokens & 15B & 50B \\
Parameter scale & 340M / 355M & 1B \\
Context / batch & 2048 / 0.5M tokens & 2048 / 0.5M tokens \\
Pairs & DN, GDN, KDA families & GDN, KDA families \\\bottomrule
\end{tabular}\end{center}\begin{itemize}
\item SlimPajama; AdamW; peak LR $4\times10^{-4}$; 8 H100 GPUs.
\item KDA/PKDA use an output gate; the smaller pair is actually 355M.
\item Compare each recurrence with its preconditioned version. Cross-family tuning differs.
\item Main accuracy tables do not report multi-seed variability.
\end{itemize}
\papersource{Sec. 4, PDF pp. 6--8; App. E.1, p. 25.}
\end{frame}

\begin{frame}{End-to-end throughput pays for the extra statistics}
\begin{center}\includegraphics[width=.94\linewidth,height=.60\textheight,keepaspectratio]{figures/fig-03.pdf}\end{center}Higher is better. Author summary: about 10\% overhead for PGDN/PKDA; PGDN more than 20\% faster than MesaNet in this profiling setup.
\papersource{Fig. 3, Sec. 4.1, PDF p. 6. 340M-scale models, 8 H100s, DDP; batch size $\times$ length on x-axis.}
\end{frame}

\begin{frame}{Two-key kernels add little cost when statistics are excluded}
\begin{center}\includegraphics[width=.94\linewidth,height=.60\textheight,keepaspectratio]{figures/fig-04.pdf}\end{center}Lower is better. Near-baseline two-key kernel timing does not remove the end-to-end cost of computing $A_t$ and $B_t$.
\papersource{Fig. 4, PDF p. 6. Forward + backward, batch 1, 8 heads, head dimension 128.}
\end{frame}

\begin{frame}{MQAR supports better resistance to associative interference}
\begin{center}\includegraphics[width=.94\linewidth,height=.60\textheight,keepaspectratio]{figures/fig-05.pdf}\end{center}Accuracy versus sequence length. Left: 64 KV pairs. Right: KV count / length = 1:8. Preconditioning generally maintains or improves accuracy in these cuts.
\papersource{Fig. 5, PDF p. 7; fixed-length KV sweep: Fig. 10, p. 26. No seed error bars shown.}
\end{frame}

\begin{frame}{All five paired task averages increase}
\begin{center}\begin{tabular}{lrr}\toprule
Pair & Commonsense avg. & Real-world ICR avg. \\\midrule
DN $\to$ PDN (340M) & $47.41\to47.96$ & $22.59\to23.84$ \\
GDN $\to$ PGDN (340M) & $48.86\to49.96$ & $24.77\to26.17$ \\
KDA $\to$ PKDA (355M) & $49.12\to49.89$ & $25.88\to26.76$ \\
GDN $\to$ PGDN (1B) & $55.44\to56.38$ & $33.87\to34.99$ \\
KDA $\to$ PKDA (1B) & $55.43\to56.64$ & $34.55\to34.88$ \\\bottomrule
\end{tabular}\end{center}
Accuracy in percent, higher is better. Averages cover 8 commonsense and 5 ICR tasks; perplexities are excluded.
\medskip
Individual tasks can regress: 1B PKDA FDA $23.23\to21.69$, SWDE $50.14\to48.79$.
\papersource{Table 1, PDF p. 7. Matched-pair interpretation; not a cross-family ranking.}
\end{frame}

\begin{frame}{Perplexity gains are larger on LAMBADA than WikiText}
\begin{center}\begin{tabular}{lrr}\toprule
Pair & LAMBADA ppl $\downarrow$ & WikiText ppl $\downarrow$ \\\midrule
DN $\to$ PDN (340M) & $45.76\to43.05$ & $29.04\to29.05$ \\
GDN $\to$ PGDN (340M) & $31.39\to28.43$ & $27.08\to26.86$ \\
KDA $\to$ PKDA (355M) & $31.37\to25.33$ & $26.18\to25.98$ \\
GDN $\to$ PGDN (1B) & $13.46\to13.09$ & $18.05\to18.01$ \\
KDA $\to$ PKDA (1B) & $14.52\to11.86$ & $17.95\to17.82$ \\\bottomrule
\end{tabular}\end{center}
1B KDA's LAMBADA perplexity drops by 2.66, or about 18.32\% relative. This is not an accuracy gain.
\medskip
The small DN WikiText regression is retained in the comparison.
\papersource{Table 1, PDF p. 7. Relative percentage calculated from reported values.}
\end{frame}

\begin{frame}{Long-context retrieval improves unevenly}
\begin{center}\includegraphics[width=.94\linewidth,height=.60\textheight,keepaspectratio]{figures/fig-06.pdf}\end{center}Trained at 2K; evaluated through 8K. Easy S-NIAH-1 largely saturates. Difficult retrieval still deteriorates with length.
\papersource{Fig. 6, PDF p. 8; exact values: Table 5, p. 27. Smaller models only in this plot.}
\end{frame}

\begin{frame}{Strong PGDN gains coexist with explicit counterexamples}
\begin{center}\begin{tabular}{llrr}\toprule
S-NIAH-3 pair & Length & Base & Preconditioned \\\midrule
GDN / PGDN (340M) & 1K & 10.4 & 89.0 \\
GDN / PGDN (340M) & 4K & 20.0 & 63.8 \\
GDN / PGDN (340M) & 8K & 3.0 & 12.8 \\
KDA / PKDA (355M) & 4K & 24.6 & 19.0 \\
GDN / PGDN (1B) & 4K & 64.2 & 63.8 \\
KDA / PKDA (1B) & 1K & 98.0 & 92.4 \\\bottomrule
\end{tabular}\end{center}
Accuracy in percent. The 340M GDN 4K gain is \alert{43.8 percentage points}.
\medskip
Do not translate the paper's broad prose into universal task-by-length dominance.
\papersource{Table 5, PDF p. 27. No multi-seed uncertainty is reported for these rows.}
\end{frame}

\begin{frame}{Trainability choices matter as much as adding curvature}
\begin{center}\begin{tabular}{lr}\toprule
340M / 15B-token configuration & Training loss $\downarrow$ \\\midrule
PGDN, bounded final form & 2.511 \\
PGDN, tied statistic/main gates & 2.518 \\
GDN & 2.528 \\
PGDN, unstable ATK & 2.536 \\
P-Mamba-2, ATQ & 2.543 \\\bottomrule
\end{tabular}\end{center}
Extra normalizations of the final preconditioner worsen loss:
$2.511$ without normalization vs. $2.517$, $2.521$, $2.524$.
\medskip
The ATQ comparison changes the base recurrence as well; it is not a universal placement theorem.
\papersource{Tables 6--7, PDF pp. 27, 30. The anomalous 5.520 entry is discussed in the notes and backup.}
\end{frame}

\begin{frame}{The mechanism evidence is suggestive, not a causal isolation}
\begin{center}\includegraphics[width=.94\linewidth,height=.49\textheight,keepaspectratio]{figures/fig-07.pdf}\end{center}\begin{itemize}
\item $k_t^\top\tilde k_t$ differs from 1, altering the write eigenvalue.
\item Observed PGDN does not learn negative eigenvalues (App. E.4).
\item Negative-spectrum baseline: GDN ICR avg. 26.22 vs. PGDN 26.17 at 340M.
\end{itemize}
\papersource{Fig. 7, PDF p. 8; Table 4, p. 26; App. E.4/Fig. 16, pp. 31, 33.}
\end{frame}

\begin{frame}{DGPS separates four practical design decisions}
\begin{center}\begin{tabular}{ll}\toprule
Axis & Decision \\\midrule
Decay & What state information is retained? \\
Gain & How strongly is the residual written? \\
Preconditioner & How are key/query directions transformed? \\
Solve & Accumulate statistics or update the fitted map? \\\bottomrule
\end{tabular}\end{center}\begin{itemize}
\item TTR supplies the regression interpretation; structured recurrences supply efficient kernels.
\item A weighted proximal norm also derives key-preconditioned updates.
\item Proposed but unevaluated variants should remain future directions.
\end{itemize}
\papersource{Sec. 5/Table 2, PDF pp. 8--9; App. B and G, pp. 15--17, 38--40.}
\end{frame}

\begin{frame}{The contribution is useful within clear evidence boundaries}
\begin{itemize}
\item A distinct, history-dependent write key is a practical lever for linear recurrent models.
\item Exact theory motivates the design; the trained model uses a bounded approximation.
\item Matched task averages improve with about 10\% reported throughput overhead.
\item No universal long-context gain, equal-compute quality frontier, or multi-seed significance claim.
\item Open questions: larger-scale training, stronger stability guarantees, and richer cheap preconditioners.
\end{itemize}
\papersource{Paper conclusions, Sec. 6, PDF p. 9; evidence boundaries synthesized from Sec. 4 and App. C/E.}
\end{frame}

\begin{frame}{Reference and reading provenance}
\textbf{Neehal Tumma, Noel Loo, Daniela Rus.}\par
\emph{Preconditioned DeltaNet: Curvature-aware Sequence Modeling for Linear Recurrences.}\par
\medskip
\href{https://arxiv.org/abs/2604.21100v1}{arXiv:2604.21100v1}, submitted 22 April 2026.\par
\medskip
Read on 22 September 2026 using the matching author TeX and 40-page PDF, including appendices A--G. Original figures are reproduced from the author's source bundle.\par
\medskip
This deck is an independent reading presentation. Experiments were not rerun. Source locators use 1-based PDF pages.

\end{frame}

\begin{frame}{Backup: exact equivalence fails after diagonal approximation}
Paper counterexample: $\lambda=1$, $k_1=v_1=(1,1)^\top$, $S_0=0$.
\[
\widehat G_1=2I,\qquad S_1^{\rm ATQ}=\tfrac12\mathbf1\mathbf1^\top.
\]
Using the previous diagonal inverse and its normalized ATK gain:
\[
\tilde k_1=\frac{I\mathbf1}{1+\mathbf1^\top I\mathbf1}=\tfrac13\mathbf1,
\qquad S_1^{\rm ATK}=\tfrac13\mathbf1\mathbf1^\top.
\]
Hence the two constructions are not generally equivalent. Special data can still make their outputs agree.
\papersource{App. C.2, PDF p. 19.}
\end{frame}

\begin{frame}{Backup: the DN--LA theorem needs the proof assumptions}
The proof additionally assumes $\lambda=0$, $v_t=S^\star k_t$ (noiseless fixed map), and $\beta_t\in[0,1]$.
Keys and independent queries are uniform on the unit sphere.
\begin{align*}
\mathbb E\|o_t^{\rm LA}-o_t^\star\|^2
&=\frac{\|S^\star\|_F^2}{d}\left(1-\frac td+\frac{t(t-1)}{d^2}\right),\\
\mathbb E\|o_t^{\rm DN}-o_t^\star\|^2
&\le\frac{\|S^\star\|_F^2}{d}.
\end{align*}
For $t\ge d+1$, the LA factor is at least 1.
This does not prove superiority of the final practical ATK model over every ATQ model.
\papersource{Theorem 3.2, PDF p. 5; assumptions and proof: App. C.5, pp. 20--21.}
\end{frame}

\begin{frame}{Backup: provenance issues should not be silently repaired}
\begin{itemize}
\item Table 6 lists GDN$[-1,1]$ training loss as \textbf{5.520} in both TeX and PDF. It is anomalous; we do not guess a correction.
\item Table 2 caption calls $G_t$ the inverse Gram; Secs. 2--3 define $G_t$ as Gram and $P_t=G_t^{-1}$. Use the equations.
\item App. E.3 sometimes describes bounding $A_t$; the actual bounded quantity is $B_t$.
\item MesaNet training failed in the authors' setup, so speed results do not establish a completed matched-quality comparison.
\end{itemize}
\papersource{Table 2, PDF p. 9; Table 6, p. 27; App. E.3/E.5, pp. 28, 31.}
\end{frame}
\end{document}