% Compile with pdflatex. Replace the demonstration content before delivery.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}

\title[Erase-then-Delta Attention]{Erase-then-Delta Attention}
\subtitle{Decoupling Erase and Write Addresses\\in Delta-Rule Linear Attention}
\author{Paper: Xiao Li et al.\\Qwen Team, Nanjing University, Zhejiang University}
\date{arXiv:2606.26560v1 (25 June 2026)\\Independent reading presentation --- 22 September 2026}
\begin{document}
\begin{frame}\titlepage\end{frame}
\begin{frame}{Erase elsewhere, then correct the current write}
\begin{block}{Main idea}Give recurrent memory an independent cleanup address before its usual delta-rule write.\end{block}\begin{itemize}
\item Reported average gains over KDA: +0.63 pp (dense), +1.07 pp (MoE) after pretraining.
\item After long-context midtraining: +0.14 pp on the general suite.
\item A conditional cleanup path; historical recall is not uniformly better.
\end{itemize}
\papersource{Paper, Eq. (8), p. 4; Tables 1--2, p. 8; Sec. 6, p. 11.}
\end{frame}

\begin{frame}{Finite-state memory must manage interference}
\begin{itemize}
\item Linear recurrence compresses history into a fixed-size state.
\item Additive writes accumulate associations; delta writes correct the current key.
\item A stale association may occupy a direction different from the new write key.
\item Motivating examples: reassignment, fact correction, context shifts. These are not separately evaluated tasks here.
\end{itemize}
\papersource{Paper, Secs. 1--2, pp. 1--3.}
\end{frame}

\begin{frame}{The delta rule corrects what the current key reads}
\[
S_t\in\mathbb R^{d_k\times d_v},\quad k_t,q_t\in\mathbb R^{d_k},\quad v_t\in\mathbb R^{d_v},\quad o_t=S_t^\top q_t
\]
\[\mathcal L_t(S)=\tfrac12\|S^\top k_t-v_t\|^2\]
\[S_t=(I-\beta_tk_tk_t^\top)D_tS_{t-1}+\beta_tk_tv_t^\top\]
\begin{itemize}
\item $D_t=\operatorname{Diag}(\alpha_t)$ retains key channels.
\item $\beta_t$ controls correction at the same address used for writing.
\item The state update is an online-gradient interpretation, not a separate model-parameter optimizer.
\end{itemize}
\papersource{Paper, Eqs. (2), (5), (7), pp. 2--3.}
\end{frame}

\begin{frame}{Gate separation and address separation differ}
\begin{tabular}{p{2.0cm}p{9.7cm}}\toprule
Variant & Memory control \\\midrule
GDN & Scalar retention within each head; delta correction at $k$.\\
KDA & Channel-wise retention; delta correction at $k$.\\
GDN-2 & Key-side erase and value-side write gates; edit still committed along $k$.\\
EDA & Independently generated $e$; erase along $e$, then correct and write along $k$.\\\bottomrule
\end{tabular}
\papersource{Paper, Sec. 2.2, p. 3; Sec. 5, pp. 10--11.}
\end{frame}

\begin{frame}{EDA preserves the delta write after cleanup}
\[
\boxed{S_t=(I-\beta_tk_tk_t^\top)(I-\gamma_te_te_t^\top)D_tS_{t-1}+\beta_tk_tv_t^\top}
\]
\begin{align*}
\widehat S_t&=D_tS_{t-1} &&\text{retain coordinates}\\
\widetilde S_t&=\widehat S_t-\gamma_te_t(\widehat S_t^\top e_t)^\top &&\text{erase a direction}\\
S_t&=\widetilde S_t+\beta_tk_t(v_t-\widetilde S_t^\top k_t)^\top &&\text{correct the write}
\end{align*}
Unit-norm $e_t,k_t$; scalar gates $\gamma_t,\beta_t\in(0,1)$.
\papersource{Paper, Eqs. (8)--(13), p. 4.}
\end{frame}

\begin{frame}{Directional erasure suppresses a memory readout}
\[
\mathcal L_t^{\rm erase}(\widehat S_t)=\tfrac12\|\widehat S_t^\top e_t\|^2
\]
\[
\widetilde S_t^\top q=\widehat S_t^\top q-\gamma_t(q^\top e_t)\widehat S_t^\top e_t
\]
\begin{itemize}
\item At $q=e_t$, the readout is multiplied by $1-\gamma_t$.
\item At $q\perp e_t$, this erase step leaves the readout unchanged.
\item Nonorthogonal queries can still change; this is not deletion of a symbolic database entry.
\end{itemize}
\papersource{Paper, Eqs. (10)--(12), p. 4.}
\end{frame}

\begin{frame}{A two-dimensional example isolates the new capability}
\[
S_{t-1}=\begin{bmatrix}3\\5\end{bmatrix},\quad e=\begin{bmatrix}1\\0\end{bmatrix},\quad
k=\begin{bmatrix}0\\1\end{bmatrix},\quad v=7
\]
Take $D=I$ and the limiting gates $\beta=\gamma=1$.
\[
\text{EDA:}\quad\begin{bmatrix}3\\5\end{bmatrix}\xrightarrow{\rm erase}\begin{bmatrix}0\\5\end{bmatrix}\xrightarrow{\rm delta}\begin{bmatrix}0\\7\end{bmatrix}
\]
\[
\text{Baseline:}\quad\begin{bmatrix}3\\5\end{bmatrix}\xrightarrow{\rm delta}\begin{bmatrix}3\\7\end{bmatrix}
\]
Expressivity illustration, not an experiment or a semantic deletion guarantee.
\papersource{Teaching example derived from Paper, Eq. (8), p. 4.}
\end{frame}

\begin{frame}{Order matters when the two addresses overlap}
\begin{align*}
&(I-\beta kk^\top)(I-\gamma ee^\top)\\
&\quad=I-\gamma ee^\top-\beta kk^\top+\beta\gamma(k^\top e)ke^\top
\end{align*}\begin{itemize}
\item The cross-term depends on address overlap.
\item Erasing after the write can also suppress newly written content.
\item Diagonal decay and directional erasure generally do not commute.
\item Reader derivation: each old-state factor is nonexpansive under normalized addresses and gates in $[0,1]$; this is not a full training-stability proof.
\end{itemize}
\papersource{Paper, Eq. (16), pp. 5--6; final bullet: reader analysis.}
\end{frame}

\begin{frame}{The extra address is generated inside the layer}
\centering\includegraphics[width=.88\linewidth,height=.69\textheight,keepaspectratio]{figures/eda.pdf}
\par Low-rank erase projection; normalized $q,k,e$; output normalization and gating.
\papersource{Paper, Fig. 1, p. 5; Appendix A, p. 13.}
\end{frame}

\begin{frame}{EDA also changes the safe decay parameterization}
\[
A=e^{A_{\log}},\quad\Delta_t=\operatorname{softplus}(a_t+b_\Delta)
\]
\[
g_t=\ell+(-\ell)\exp\!\left(-\frac{A}{|\ell|}\odot\Delta_t\right),\qquad\alpha_t=e^{g_t}
\]
\begin{itemize}
\item With $\ell=-5$: $e^{-5}<\alpha_{t,i}\leq1$; near zero, $g\approx-A\Delta$.
\item Large decay inputs saturate instead of sending log-retention to $-\infty$.
\item Important attribution limit: reported EDA/KDA comparisons also differ in safe gate and parameter count.
\end{itemize}
\papersource{Paper, Eqs. (14)--(15), p. 5; Appendix A, pp. 13--14.}
\end{frame}

\begin{frame}{Two virtual steps recover a standard delta recurrence}
\centering
\begin{tabular}{lccccc}\toprule
Sub-step & $q'$ & $k'$ & $v'$ & $\beta'$ & $\alpha'$\\\midrule
Erase ($2t-1$) & $0$ & $e_t$ & $0$ & $\gamma_t$ & $\alpha_t$\\
Delta ($2t$) & $q_t$ & $k_t$ & $v_t$ & $\beta_t$ & $\mathbf1$\\\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Odd steps decay and erase without writing a value.
\item Even steps correct/write and produce the real output.
\item WY / UT chunk formulations aim to turn recurrence into matrix operations.
\item Twice as many sub-steps increases prefill work, while state size is unchanged.
\end{itemize}
\papersource{Paper, Sec. 3.4, Eqs. (17)--(22), pp. 6--7.}
\end{frame}

\begin{frame}{Efficiency is argued, not quantitatively benchmarked}
\begin{itemize}
\item New kernel inputs: erase address $e$ and scalar gate $\gamma$.
\item Authors argue fusion keeps extra HBM traffic modest and decoding remains state-I/O dominated.
\item No latency, throughput, hardware sweep, or measured overhead table is provided.
\item Fixed-size recurrent memory does not imply fixed memory for the whole 3:1 hybrid model.
\end{itemize}
\papersource{Paper, Sec. 3.5 and Sec. 4.1, p. 7; last bullet: architectural implication.}
\end{frame}

\begin{frame}{Evaluation uses matched training within two model families}
\begin{tabular}{lcc}\toprule
 & Dense 2.5B & MoE 25B-A2.8B\\\midrule
Layers (linear/full) & 24 (18/6) & 28 (21/7)\\
Model width & 2048 & 2048\\
Pretraining tokens & 400B & 400B\\
Sequence / global batch & 4096 / 1024 & 4096 / 1024\\
Extra midtraining & -- & 80B at 32k\\\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item All recurrent variants: three linear layers, then one full-attention layer.
\item Dense includes Transformer, GDN, GDN-2, KDA, EDA; MoE includes GDN, KDA, EDA.
\item BF16, AdamW; parameter counts are close but not identical.
\end{itemize}
\papersource{Paper, Sec. 4.1, p. 7; Tables 4--5, pp. 13--14.}
\end{frame}

\begin{frame}{Metrics mix knowledge, reasoning, and code}
\begin{itemize}
\item MMLU / MMLU-Pro: broad knowledge and harder multiple-choice questions.
\item GSM8K / MATH: mathematical reasoning; BBH: difficult reasoning tasks.
\item EvalPlus: code generation under stricter functional tests.
\item Tables 1--2: percentages, two evaluations of the same checkpoint; Avg. is the unweighted six-task mean.
\item No independent-training-seed variance or confidence intervals are reported.
\end{itemize}
\papersource{Paper, Sec. 4.1, p. 7; Appendix B, p. 14.}
\end{frame}

\begin{frame}{Dense EDA leads the mean, not every benchmark}
\centering\begin{tabular}{lrrrr}\toprule
Model & MMLU & GSM8K & EvalPlus & Six-task Avg.\\\midrule
Transformer & 50.11 & 20.26 & 30.82 & 27.75\\
GDN & 49.99 & 20.83 & 31.08 & 27.58\\
GDN-2 & 49.90 & 21.40 & 32.95 & 28.14\\
KDA & 50.03 & 21.30 & 30.73 & 27.81\\
EDA & 49.27 & 23.90 & 32.50 & \textbf{28.44}\\\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item EDA vs. KDA: +0.63 pp overall; +2.60 pp GSM8K, but $-0.76$ pp MMLU.
\item Only selected task columns shown; Avg. still uses all six tasks.
\end{itemize}
\papersource{Paper, Table 1, p. 8. Percentages; higher is better; 400B tokens.}
\end{frame}

\begin{frame}{MoE EDA leads five of six pretraining tasks}
\centering\begin{tabular}{lrrrr}\toprule
Model & GSM8K & MATH & EvalPlus & Six-task Avg.\\\midrule
GDN & 58.89 & 31.87 & 50.09 & 48.16\\
KDA & 58.61 & 30.78 & 51.60 & 48.31\\
EDA & 57.71 & 33.57 & 53.37 & \textbf{49.38}\\\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item EDA vs. KDA: +1.07 pp overall; MATH +2.79 pp, EvalPlus +1.77 pp.
\item GSM8K is the exception: EDA trails both baselines.
\item Evidence across two settings, not a fitted scaling law.
\end{itemize}
\papersource{Paper, Table 1, p. 8. Percentages; higher is better; 400B tokens.}
\end{frame}

\begin{frame}{Midtraining retains only a small mean advantage}
\centering\begin{tabular}{lrrrr}\toprule
Model & GSM8K & MATH & EvalPlus & Six-task Avg.\\\midrule
GDN & 75.93 & 45.94 & 50.09 & 57.37\\
KDA & 76.21 & 46.87 & 53.82 & 58.31\\
EDA & 75.99 & 49.28 & 51.78 & \textbf{58.45}\\\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item EDA vs. KDA: +0.14 pp overall, down from +1.07 pp before midtraining.
\item MATH +2.41 pp is partly offset by EvalPlus $-2.04$ pp.
\item Reported means do not establish statistical significance.
\end{itemize}
\papersource{Paper, Table 2, p. 8. MoE: 400B pretraining + 80B tokens at 32k.}
\end{frame}

\begin{frame}{RULER gains are strongest at 4k--16k}
\centering\begin{tabular}{lrrrrrrr}\toprule
Model & 4k & 8k & 16k & 32k & 64k & 128k & Avg.\\\midrule
GDN &92.40&90.09&87.28&82.15&67.62&45.16&77.45\\
KDA &93.33&89.29&85.13&79.66&72.00&42.70&77.02\\
EDA &93.84&90.88&87.55&81.52&71.90&44.22&78.32\\\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item EDA has the highest six-length average, not the best score at every length.
\item 64k / 128k extrapolate beyond 32k midtraining; all models degrade substantially.
\item Four evaluation runs; averages across subtasks and displayed lengths.
\end{itemize}
\papersource{Paper, Table 3, p. 9. Percentages; higher is better.}
\end{frame}

\begin{frame}{The erase path receives more budget when decay is weak}
\[
b_D=1-\bar\alpha,\quad b_\Delta=\bar\alpha\beta,\quad b_E=\bar\alpha\gamma,\quad q_m=\frac{b_m}{b_D+b_\Delta+b_E}
\]
\begin{itemize}
\item Global gate shares: decay 35.4\%, delta 31.8\%, erase 32.8\%.
\item For $\bar\alpha\geq0.9$, erase share is 69.1\%, about $3\times$ delta.
\item These are gate-strength shares, not fractions of removed state energy.
\item State projections, anisotropy, and address overlap are not accounted for.
\end{itemize}
\papersource{Paper, Sec. 4.5, Eqs. (23)--(24), pp. 9--10; Fig. 2(a).}
\end{frame}

\begin{frame}{Learned addresses perturb current reads less}
\centering\includegraphics[width=\linewidth,height=.51\textheight,keepaspectratio]{figures/eda_memory_state_analysis.pdf}
\par\medskip
Actual / random / shuffled / same-key raw perturbation means:\\
$0.064\;/\;0.143\;/\;0.115\;/\;0.223$.
\par\smallskip Layer-normalized ratios in panel (b) are not ratios of raw means.
\papersource{Paper, Fig. 2, p. 10; same erase gates, counterfactual direction replay.}
\end{frame}

\begin{frame}{The probes support decoupling, not causal task benefit}
\begin{itemize}
\item Mean $|\cos(e,k)|\approx0.105$: far from same-address collapse at 1.
\item Independent random directions in 128D already have mean $|\cos|\approx0.0707$.
\item Low perturbation is not a direct measurement of deleting stale facts or improving predictions.
\item KDA performs better on raw write-key recall; the paper gives no numerical recall curve.
\end{itemize}
\papersource{Paper, Sec. 4.5, pp. 9--10; Sec. 6, p. 11. Random-reference value: reader calculation.}
\end{frame}

\begin{frame}{A useful update rule with clear evidence boundaries}
\begin{itemize}
\item Contribution: independently addressed cleanup while retaining corrective delta writing.
\item Evidence: better reported task averages in the tested hybrid settings.
\item Limits: different safe gate, small parameter increases, no seed uncertainty, no timed kernels.
\item Next tests: matched-gate ablations, controlled invalidation tasks, causal interventions, and recurrence/kernel equivalence.
\end{itemize}
\papersource{Paper, Sec. 6, p. 11; limitations and next tests include reader analysis.}
\end{frame}

\begin{frame}{Primary source and discussion route}
\textbf{Xiao Li et al.}\\
Erase-then-Delta Attention: Decoupling Erase and Write Addresses in Delta-Rule Linear Attention.\\
\href{https://arxiv.org/abs/2606.26560v1}{arXiv:2606.26560v1}, 25 June 2026.
\medskip
\begin{itemize}
\item Read Eq. (8) for the update, Table 3 for long-context boundaries, and Sec. 4.5 for probe interpretation.
\item This presentation uses the same-version author TeX, PDF, and original figures.
\item Discussion: when should finite memory sacrifice raw historical recall for current prediction?
\end{itemize}
\papersource{Independent reading presentation; no pretraining or kernel performance reproduction.}
\end{frame}

\begin{frame}{Backup: printed chunk formulas need implementation checks}
Paper Eq. (21) prints
\[
M=I+\operatorname{StrictTril}(\cdots),\quad W=M(A\odot K'),\quad U=MV'.
\]
\begin{itemize}
\item Under ordinary ungated $K',V'$ notation, a single sub-step gives $M=I$ and $U=V'$; Eq. (20) instead requires the $\beta'$ factor.
\item A triangular-solve / inverse convention and gate placement need clarification; matrix layout also needs checking.
\item Do not silently repair the paper. Validate an implementation against unambiguous Eqs. (8), (17)--(18).
\end{itemize}
\papersource{Reader consistency check against Paper, Eqs. (20)--(22), pp. 6--7; not an author correction.}
\end{frame}

\begin{frame}{Backup: parameter differences are small but nonzero}
\centering\begin{tabular}{llrr}\toprule
Scale & Model & Total (B) & Active (B)\\\midrule
Dense & KDA & 2.5218 & 2.5218\\
Dense & EDA & 2.5295 & 2.5295\\
MoE & KDA & 24.6324 & 2.7885\\
MoE & EDA & 24.6558 & 2.8119\\\bottomrule
\end{tabular}
\medskip
\begin{itemize}
\item Dense increase: 7.7M, approximately 0.31\% total parameters.
\item MoE increase: 23.4M, approximately 0.84\% active parameters.
\item MoE erase projection: rank 16 per head; safe log-decay bound $-5$.
\end{itemize}
\papersource{Paper, Table 5, p. 14; Appendix A, p. 13. Percent changes: reader calculations.}
\end{frame}
\end{document}