% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}


\title[OSDN]{OSDN: Online Preconditioning\\inside the Delta Rule}
\subtitle{A paper reading: mechanism, guarantees, and evidence}
\author{Paper: Chenyu Zhou, Hongpei Li, Yuerou Liu,\\Jianghao Lin, Dongdong Ge, Yinyu Ye}
\date{arXiv:2605.13473v1 (13 May 2026)\\Reading: 22 September 2026}
\begin{document}
\begin{frame}\titlepage\end{frame}
\begin{frame}{A small write-side change has a specific payoff}
\begin{itemize}
\item OSDN learns a diagonal write multiplier online from keys and scalar gates.
\item At 340M: JRT recall rises from 0.150 to 0.198 (+32\% relative).
\item At 1.3B: local residual ratio falls by 39\%; repeated recall does not improve.
\item Central question: how can a preconditioner remain compatible with chunkwise DeltaNet?
\end{itemize}
\papersource{Paper, Secs.4--6; Tables 1 and 8, PDF pp.7 and 35. Main talk: approximately 25 minutes.}
\end{frame}

\begin{frame}{DeltaNet writes an error, using one scalar step}

\[
S_{t-1}\in\mathbb R^{V\times K},\quad k_t,q_t\in\mathbb R^K,\quad v_t\in\mathbb R^V
\]
\[
u_t=v_t-S_{t-1}k_t,\qquad S_t=S_{t-1}+\beta_tu_tk_t^\top
\]
\begin{block}{An inner regression step}
\(f_t(S)=\tfrac12\|Sk_t-v_t\|^2\), so \(\nabla f_t(S_{t-1})=-u_tk_t^\top\).
\end{block}
\begin{itemize}
\item The learned gate $\beta_t\in(0,1)$ is shared across key coordinates.
\item Repeated, rare, and distracting directions may need different write scales.
\end{itemize}
\papersource{Paper, Secs.1, 3 and 4.1, PDF pp.1--5.}
\end{frame}

\begin{frame}{Diagonal right-preconditioning scales only the write key}

\[
D_t=\operatorname{Diag}(d_t),\qquad \widetilde k_t=d_t\odot k_t
\]
\begin{align*}
S_t&=S_{t-1}-\beta_t\nabla f_t(S_{t-1})D_t\\
   &=S_{t-1}+\beta_t\underbrace{(v_t-S_{t-1}k_t)}_{\text{original read key}}\,
         \underbrace{\widetilde k_t^\top}_{\text{scaled write key}}.
\end{align*}
\begin{itemize}
\item Readout remains $o_t=S_tq_t$; the memory keeps its $V\times K$ shape.
\item Extra persistent state per head: $d_t\in\mathbb R^K$.
\item Scaling both keys would define a different recurrence.
\end{itemize}
\papersource{Paper, Sec.4.1, Eq.(2), PDF pp.4--5.}
\end{frame}

\begin{frame}{The residual cancels out of the hypergradient}

Let \(s_t=k_t^{\odot2}\), \(n_t=\|k_t\|^2\), and \(a_t=\beta_t\langle d_t,s_t\rangle\).
\[
S_tk_t-v_t=-u_t(1-a_t)
\]
\[
h_t(d_t)=\frac{f_t(S_t)-f_t(S_{t-1})}{\|\nabla f_t(S_{t-1})\|_F^2}
        =\frac{(1-a_t)^2-1}{2n_t}
\]
\[
\nabla_dh_t(d_t)=-\frac{\beta_t}{n_t}(1-a_t)s_t
\]
\begin{block}{Why this matters}
The feedback needs keys and gates, but no value, memory matrix, or residual.
\end{block}

\papersource{Paper, Lemma 4.1 / Eq.(3), PDF p.5; App.A.3, pp.15--16. Derivation assumes nonzero key and residual.}
\end{frame}

\begin{frame}{Emit the current write key before updating the scale}

\begin{enumerate}
\item Emit \(\widetilde k_t=d_t\odot k_t\).
\item Compute \(s_t=k_t^{\odot2}\) and \(\bar n_t=\max(\|k_t\|^2,\epsilon)\).
\item Update
\[
d_{t+1}=\Pi_{\mathcal D}\!\left[d_t+\eta\beta_t
          \frac{1-\beta_t\langle d_t,s_t\rangle}{\bar n_t}s_t\right].
\]
\end{enumerate}
\begin{block}{Reported 340M final configuration}
\(d_1=\mathbf1\), \(\eta=0.003\), \(\mathcal D=[0.5,2]^K\).
\end{block}
With unit keys, \(0<a_t<2\), hence the local loss ratio \((1-a_t)^2<1\).

\papersource{Paper, Algorithm 1 / Eq.(4), PDF p.5; Cor.A.2, p.16; App.F, p.32.}
\end{frame}

\begin{frame}{Decoupling preserves the expensive chunk computation}

\begin{block}{Phase 1: a lightweight recurrent sweep}
\((k_t,\beta_t,d_t)\longrightarrow\widetilde k_t, d_{t+1}\).
\quad Work/token: \(O(K)\); persistent state/head: \(O(K)\).
\end{block}
\begin{block}{Phase 2: the asymmetric DeltaNet write}
\[
S_t=S_{t-1}(I-\beta_tk_t\widetilde k_t^\top)+\beta_tv_t\widetilde k_t^\top
\]
Chunk Gram: \(BKK^\top\to BK\widetilde K^\top\).\quad
Local score: \(QK^\top\to Q\widetilde K^\top\).
\end{block}
Matrix shapes and triangular solves are retained. Phase 1 still has sequence-length sequential depth in Table 2; clipping makes it piecewise affine.

\papersource{Paper, Secs.4.3--4.5, PDF pp.5--6; Table 2, p.17; App.C, pp.20--23.}
\end{frame}

\begin{frame}{APF forgets calibration in the preconditioner}

\[
r_{t,h}=\sigma(w_{r,h}^\top x_t+b_{r,h})
\]
\[
d_{t+1}=\Pi_{\mathcal D}\!\left[r_{t,h}d_t+\eta\beta_t
\frac{1-\beta_t\langle d_t,s_t\rangle}{\bar n_t}s_t\right]
\]
\begin{itemize}
\item A token-wise, head-wise retention gate responds to changing contexts.
\item It acts on $d_t$; the memory $S_t$ is not directly decayed by APF.
\item Adds $H(d_m+1)$ parameters/layer; reported as at most 0.3\% at 340M.
\item The current super-geometric analysis does not directly cover APF.
\end{itemize}
\papersource{Paper, Sec.4.4 / Eq.(7), PDF p.6; App.A.5, p.16; Discussion, p.9.}
\end{frame}

\begin{frame}{Two guarantees control different mathematical objects}
\textbf{Population objective: ideal full-gradient dynamics}
\par\smallskip
\(f=\mathbb E[f_t]\), \(\Sigma_k=\mathbb E[kk^\top]\), \(L=\lambda_{\max}(\Sigma_k)\).
\[
f(S_{T+1})-f^\star\le[f(S_1)-f^\star]
\left(\frac{2L\mathcal R_T}{T}\right)^T
\]
Monotone updates; cumulative regret \(\mathcal R_T\) against \(D_\star=\Sigma_k^\dagger\).
\par\medskip
\textbf{Token-local objective: diagonal surrogate}
\[
\prod_{t=1}^T\frac{f_t(S_t)}{f_t(S_{t-1})}
\le\left(2\varepsilon_T^{\rm diag}+\frac{2R_T}{T}\right)^T
\]
Unit keys; diagonal regret \(R_T\); fixed-comparator gap \(\varepsilon_T^{\rm diag}\).
\papersource{Paper, Theorems 5.1--5.2, PDF pp.6--7; detailed proofs in App.D.}
\end{frame}

\begin{frame}{Super-geometric contraction is conditional}

\[
\varepsilon_T^{\rm diag}=\min_{d\in\mathcal D}\frac1{2T}
\sum_t(1-\beta_t\langle d,k_t^{\odot2}\rangle)^2
\]
\begin{itemize}
\item Zero comparator gap requires one feasible $d^\star$ satisfying $\beta_t\langle d^\star,k_t^{\odot2}\rangle=1$ for every token.
\item Together with $R_T=O(\sqrt T)$, the bound becomes $(O(1)/\sqrt T)^T$.
\item The practical fixed step $\eta=0.003$ has no dedicated regret derivation here.
\item Reader inference: if $\beta_t<0.5$ and $d_j\le2$, exact matching is impossible at that token.
\end{itemize}
\papersource{Paper, Thm.5.2, p.7; App.D.4--D.5, pp.28--31; Discussion, p.9.}
\end{frame}

\begin{frame}{Perfect local writes can still overwrite useful memory}

\begin{block}{The paper's counterexample}
\(K=V=1\), \(k_t=1\), \(v_t=(-1)^t\), and an exact local write.
\[
f_t(S_t)=0,\qquad S_t=(-1)^t,\qquad S^\star_{\rm population}=0.
\]
\end{block}
\begin{itemize}
\item Consecutive losses $f_t$ differ; their ratios do not telescope into a global loss.
\item A repeated-key corollary needs fixed targets, disjoint key supports, and compatible blocks.
\item Natural-language repeated prompts are not shown to satisfy those assumptions.
\end{itemize}
\papersource{Paper, App.D.4, counterexample and Cor.D.8, PDF pp.29--30.}
\end{frame}

\begin{frame}{Evaluation separates recall, mechanism, and broad capability}

\begin{tabular}{@{}p{0.23\linewidth}p{0.69\linewidth}@{}}
\toprule
Question & Measurement \\
\midrule
Recall & JRT contains accuracy at 2K; single/repeated FDA, SWDE, SQuAD \\
Write mechanism & Residual-ratio geometric mean on twice prompts \\
Broad capability & LM PPL; 8-task commonsense; 14-task LongBench \\
Length extrapolation & PG-19 20K blocks, 2K position buckets \\
Cost & H100 prefill + decode throughput; state count \\
\bottomrule
\end{tabular}\par
\medskip
340M: 10.74B tokens, fixed seed 42, matched within each backbone. Packed 64K tensors contain segments capped at 4K with state resets.

\papersource{Paper, Sec.6 and Apps.E--F, PDF pp.7--8 and 31--34. The 2K training claim elsewhere conflicts with App.F.}
\end{frame}

\begin{frame}{340M: recall gains are strongest on repeated contexts}

\centering
\begin{tabular}{lrrrr}
\toprule
Model & All $\uparrow$ & Single $\uparrow$ & Repeated $\uparrow$ & LM PPL $\downarrow$\\
\midrule
DeltaNet & 0.150 & 0.155 & 0.145 & 32.00\\
OSDN & \textbf{0.198} & \textbf{0.179} & \textbf{0.218} & 31.67\\
OSDN-APF & 0.176 & 0.152 & 0.199 & \textbf{30.99}\\
\bottomrule
\end{tabular}\par
\medskip
\begin{block}{Interpret the headline correctly}
Overall: +4.8 percentage points, or +32\% relative.\\
Repeated: +7.3 points, or about +50\% relative.
\end{block}
\raggedright
Gains vary by task: OSDN SQuAD-twice is 0.231\(\to\)0.383; FDA-twice is 0.045\(\to\)0.044.

\papersource{Paper, Table 1, PDF p.7; Table 7, p.34. LM PPL = WikiText/LAMBADA geometric mean.}
\end{frame}

\begin{frame}{The residual diagnostic supports the intended mechanism}

\centering
\includegraphics[width=0.96\linewidth,height=0.54\textheight,keepaspectratio]{figures/residual-ratio.pdf}
\par\medskip\raggedright
\(q_{\rm geo}\): 0.537\(\to\)0.433 (OSDN), 0.425 (APF).\\
48 prompts; 24 layers; 8 heads; about 7.85 million correlated measurements. Lower curves show relative improvement, not uniformly lower values after repetition.

\papersource{Paper, original Fig.2, PDF p.9; Table 8, p.35. Scalar residual ratio q is distinct from the query vector.}
\end{frame}

\begin{frame}{The same intervention has different backbone tradeoffs}

\centering
\begin{tabular}{lrrr}
\toprule
Model & Repeated recall $\uparrow$ & LM PPL $\downarrow$ & LongBench $\uparrow$\\
\midrule
GDN & 0.139 & 30.01 & 0.073\\
OSGDN & 0.195 & 29.78 & 0.073\\
OSGDN-APF & 0.221 & 29.50 & 0.080\\
\midrule
KDA & 0.150 & \textbf{26.75} & 0.088\\
OSKDA & 0.133 & 27.93 & 0.090\\
OSKDA-APF & 0.179 & 28.02 & 0.098\\
\bottomrule
\end{tabular}\par
\medskip\raggedright
GDN benefits on repeated recall. KDA variants trade language-model perplexity against selected recall and LongBench gains.

\papersource{Paper, Table 1, PDF p.7; Tables 6--7, p.34. Comparisons are within each host family.}
\end{frame}

\begin{frame}{At 1.3B, stronger local contraction does not lift repeated recall}

\centering
\begin{tabular}{lrr}
\toprule
Metric & DeltaNet & OSDN-APF\\
\midrule
Residual $q_{\rm geo}\downarrow$ & 0.432 & \textbf{0.265}\\
Single recall $\uparrow$ & 0.293 & 0.315\\
Repeated recall $\uparrow$ & 0.227 & 0.217\\
LM PPL $\downarrow$ & 14.28 & 14.22\\
Common. $\uparrow$ & 0.560 & 0.566\\
LongBench $\uparrow$ & 0.115 & 0.116\\
\bottomrule
\end{tabular}\par
\medskip\raggedright
Recall averages FDA/SWDE only; SQuAD accuracy is excluded after harness failures. No matched PG-19 result. No multi-seed interval establishes equivalence.

\papersource{Paper, Table 8, PDF p.35; Tables 15--18, p.40. 1.3B / 100B tokens.}
\end{frame}

\begin{frame}{The 1.3B update differs from the main theoretical algorithm}

\begin{block}{340M beta-aware phase 1, before projection}
\[
d_{t+1}=r_td_t+\eta\beta_t(1-\beta_t\langle d_t,k_t^2\rangle)k_t^2
\]
\end{block}
\begin{block}{Reported 1.3B non-beta-aware phase 1}
\[
d_{t+1}=\rho_t^dd_t+\eta(1-\langle d_t,k_t^2\rangle)k_t^2
\]
\end{block}
Unit-key form shown. The memory write still uses \(\beta_t\).\\
The 1.3B configuration also uses a learned initial scale and retention. Its empirical result is not a direct verification of the beta-aware, unforgotten theorem.

\papersource{Paper, App.B, PDF p.17; App.C, p.20; theoretical scope in Discussion, p.9.}
\end{frame}

\begin{frame}{PG-19: lowest final PPL is not lowest late-context drift}

\centering
\includegraphics[width=0.90\linewidth,height=0.53\textheight,keepaspectratio]{figures/pg19.pdf}
\par\smallskip\raggedright
DeltaNet / OSDN / APF: final PPL = 20.78 / 20.02 / 19.85.\\
Late drift = +1.69 / +1.19 / +2.13. APF has the lowest final PPL, while vanilla OSDN has the smallest drift within this family.

\papersource{Paper, original Fig.4 and Table 13, PDF p.38. Drift = PPL at 18--20K minus PPL at 2--4K.}
\end{frame}

\begin{frame}{Ablations and broad checks narrow the claim}

\centering
\begin{tabular}{lrrr}
\toprule
Variant & Recall $\uparrow$ & PG-19 PPL $\downarrow$ & Late drift $\downarrow$\\
\midrule
OSDN & 0.198 & 20.02 & +1.19\\
OSDN + learned $D_0$ & 0.172 & 21.57 & +5.34\\
OSDN + $D_0^\star$ & 0.164 & 21.42 & +4.25\\
OSDN-APF & 0.176 & 19.85 & +2.13\\
\bottomrule
\end{tabular}\par
\medskip\raggedright
\begin{itemize}
\item Learning the initial scale does not recover the best recall or length profile.
\item DROP/NQ/TriviaQA averages are all 0.016 for the DeltaNet family.
\item Training CE spans only 0.058 nats across completed runs; it is a different loss from the theorem.
\end{itemize}
\papersource{Paper, Table 14, p.38; Table 12, p.37; Fig.5, p.39. $D_0^\star$: relaxed initial-scale projection.}
\end{frame}

\begin{frame}{Inference overhead is modest, with a state-percentage error}

\centering
\begin{tabular}{lrrr}
\toprule
Model & tokens/s $\uparrow$ & Change & State MiB\\
\midrule
DeltaNet 340M & 784.5 & --- & 6.000\\
OSDN 340M & 782.7 & $-0.2\%$ & 6.047\\
OSDN-APF 340M & 767.2 & $-2.2\%$ & 6.047\\
DeltaNet 1.3B & 794.0 & --- & 12.000\\
OSDN-APF 1.3B & 739.9 & $-6.8\%$ & 12.094\\
\bottomrule
\end{tabular}\par
\medskip\raggedright
Single H100; batch 1; 2048-token prefill + 128-token decode.\\
\textbf{Reader calculation:} \(0.047/6.000\approx0.78\%\), inconsistent with the paper's ``at most 0.05\%'' claim. Counts use bf16 and exclude activations and short-conv cache.

\papersource{Paper, Tables 19--20 and state-count formula, PDF pp.41--42. Throughput includes prefill; not decode-only.}
\end{frame}

\begin{frame}{What the evidence establishes, and what remains open}
\begin{itemize}
\item \textbf{Established algebra:} diagonal right-preconditioning is an asymmetric write-key substitution with decoupled feedback.
\item \textbf{Positive evidence:} 340M controlled recall gains and smaller local residual ratios at both scales.
\item \textbf{Limits:} conditional theory, APF and scale-specific update gaps, one seed, missing 1.3B evaluations, mixed broad-task results.
\item \textbf{Next test:} match beta-aware/non-beta-aware and APF/no-APF runs at 1.3B; measure retained associations as well as local write errors.
\end{itemize}
\papersource{Paper, Discussion, PDF pp.9--10; Apps.B--L. Proposed next test is reader analysis.}
\end{frame}

\begin{frame}{Reference and reading materials}

\textbf{Chenyu Zhou, Hongpei Li, Yuerou Liu, Jianghao Lin, Dongdong Ge, and Yinyu Ye.}
\medskip

\emph{OSDN: Improving Delta Rule with Provable Online Preconditioning in Linear Attention.}
\medskip

\href{https://arxiv.org/abs/2605.13473v1}{arXiv:2605.13473v1}, 13 May 2026.
\medskip

This presentation is a reading of the paper, prepared on 22 September 2026. It is not an author presentation or an independent experimental reproduction.
\medskip

Companion: Chinese reading notes, original figure assets, and editable Beamer source in this reading directory.

\papersource{Source coverage: main text and Appendices A--M; PDF and matching author TeX source.}
\end{frame}

\begin{frame}{Backup: the token-local proof is regret plus AM--GM}

For nonzero pre-write residuals and unit keys,
\[
q_t=(1-\beta_t\langle d_t,s_t\rangle)^2=1+2h_t(d_t).
\]
For any fixed comparator \(d\in\mathcal D\),
\begin{align*}
\frac1T\sum_t q_t
&=1+\frac2T\sum_t h_t(d_t)\\
&\le1+\frac2T\left(\sum_t h_t(d)+R_T(d)\right)\\
&=2\varepsilon_T(d)+\frac{2R_T(d)}T.
\end{align*}
Apply \(\prod_tq_t\le(\frac1T\sum_tq_t)^T\).

\papersource{Paper, Thm.D.7 proof, PDF p.29. Zero-residual conventions need separate treatment.}
\end{frame}

\begin{frame}{Backup: the chunk matrices keep the read/write asymmetry}

For a chunk of \(C\) tokens, \(K,\widetilde K\in\mathbb R^{C\times K}\),
\(V\in\mathbb R^{C\times V}\), and \(B=\operatorname{Diag}(\beta)\):
\[
T=\left[I+\operatorname{tril}(BK\widetilde K^\top,-1)\right]^{-1}
\]
\[
W=TBK,\qquad U=TBV
\]
\[
S_{\rm next}=S+(U-WS^\top)^\top\widetilde K.
\]
The physical key remains in \(W\). The scaled key appears in storage interactions and the final write. Materializing \(\widetilde K\) also consumes temporary sequence storage.

\papersource{Paper, App.A.2, PDF p.15; Listing 1, pp.20--22.}
\end{frame}

\begin{frame}{Backup: GDN and KDA scale the same storage factor}

\textbf{GDN:} apply its state gate first,
\[
\bar S=\alpha_tS_{t-1},\quad
u_t=v_t-\bar Sk_t,\quad S_t=\bar S+\beta_tu_t\widetilde k_t^\top.
\]
\textbf{KDA:} use its native transposed state, \(S_t\in\mathbb R^{K\times V}\),
\[
\bar S=\operatorname{Diag}(\alpha_t)S_{t-1},\quad
u_t=v_t-\bar S^\top k_t,
\]
\[
S_t=\bar S+\beta_t\widetilde k_tu_t^\top.
\]
The state-retention gate and preconditioner-retention gate have distinct roles. Post-gate residuals preserve the local feedback derivation.

\papersource{Paper, App.B and Table 3, PDF pp.17--19.}
\end{frame}

\begin{frame}{Backup: training cross-entropy is a separate diagnostic}

\centering
\includegraphics[width=0.91\linewidth,height=0.65\textheight,keepaspectratio]{figures/training-loss.pdf}
\par\raggedright
257-step centered smoothing; final values average 256 steps. These curves do not establish a super-geometric language-model loss rate.

\papersource{Paper, original Fig.5, PDF p.39. Full panel has log y-axis; late panels have linear y-axes.}
\end{frame}

\begin{frame}{Backup: reporting details matter for reproduction}
\begin{itemize}
\item App.E says 2K training context; App.F and Fig.4 specify segments capped at 4K.
\item App.C mentions OSKDA retention 0.999; App.F labels refreshed final OSKDA as no-retention.
\item Table 9 selects the best completed no-APF cell per task before averaging commonsense results.
\item APF commonsense is 0.454 in the ablation but 0.456 in the main results.
\item Fig.1 uses $d_{t+1}$ for the write key; Algorithm 1 emits $d_t\odot k_t$ before updating.
\end{itemize}
\papersource{Paper, PDF pp.2, 5, 23, 32, 35 and 38. Reading follows explicit final configurations and algorithm ordering.}
\end{frame}
\end{document}