% Self-authored discussion slides, based on the skill's Beamer template.
% Build: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\setbeamertemplate{frametitle}{%
  \vskip2mm\usebeamerfont{frametitle}\usebeamercolor[fg]{frametitle}%
  \strut\insertframetitle\strut\par\vskip1mm}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\D}{\operatorname{Diag}}
\newcommand{\sgm}{\operatorname{sigmoid}}
\newcommand{\base}{\mathrm{GDN2}}
\newcommand{\qedm}{\mathrm{QED}}
\title[QED]{The Query Knows What to Forget:\\A Second Erase Direction for Linear Attention}
\author{Paper authors: Dhruman Gupta, Aritra Das, Debayan Gupta}
\date{arXiv:2608.13668v1 \quad | \quad Discussion prepared 2026-10-30}

\begin{document}

% 1
\begin{frame}
  \vspace{0pt}
  {\usebeamerfont{title}\usebeamercolor[fg]{title}\inserttitle\par}
  \vspace{5mm}
  {\large Query-derived Erase Direction (QED)}\par
  \vspace{4mm}
  Dhruman Gupta, Aritra Das, Debayan Gupta\\
  Truth Audit Labs\par
  \vspace{4mm}
  \href{https://arxiv.org/abs/2608.13668v1}{arXiv:2608.13668v1} \quad Submitted 2026-08-13\par
  \vspace{4mm}
  {\small Publication not found as of 2026-10-30.\\
  Public code not found as of 2026-10-30.}
  \vspace{4mm}
  \papersource{Research group discussion, prepared 2026-10-30; not an author presentation.}
\end{frame}

% 2
\begin{frame}{The gain is retrieval at a fixed state size}
  \begin{block}{Idea}
    Use the query to select old content for an erase correction, while
    keeping a single rank-one state update.
  \end{block}
  \begin{itemize}
    \item 340M parameters, 15B training tokens, a 2K training window.
    \item At 8K, S-NIAH-1 improves by \alert{27.2 / 33.6 points} across two seeds.
    \item Maximum gain: \alert{38.6 points} at 16K, seed 1.
    \item Validation loss and SQuAD-completion remain nearly unchanged.
  \end{itemize}
  \papersource{Paper, Secs. 3--6; Tables 2--3, PDF pp. 5, 7. Differences are percentage points.}
\end{frame}

% 3
\begin{frame}{Many associations share the same recurrent state}
  \[
    S_t\in\mathbb R^{K\times V},\qquad
    o_t=S_t^\top q_t
       =\sum_{j\le t}(q_t^\top\tilde k_j)\tilde v_j
  \]
  \begin{itemize}
    \item More context adds associations without enlarging the state.
    \item The query determines which stored associations enter the read.
    \item Existing delta rules select old content through a key-derived erase vector.
  \end{itemize}
  \begin{block}{Design question}
    Can the read direction help determine the state correction?
  \end{block}
  \papersource{Paper, Secs. 1--3.1, Eq. (5), PDF pp. 1--3. Tildes denote transformed stored pairs.}
\end{frame}

% 4
\begin{frame}{GDN-2 separates decay, erase, and write}
  \[
    \bar S_t=\D(\exp g_t)S_{t-1},\qquad a_t=b_t\odot k_t
  \]
  \[
    S_t=\bar S_t+
      k_t\underbrace{\left[w_t\odot v_t-\bar S_t^\top a_t\right]^\top}_{\text{residual to write}},
    \qquad o_t=S_t^\top q_t
  \]
  \begin{itemize}
    \item $q_t,k_t,b_t,g_t\in\mathbb R^K$; $v_t,w_t\in\mathbb R^V$.
    \item $q_t,k_t$ are unit vectors; $b_t,w_t$ are channel-wise gates in $(0,1)$.
    \item $a_t$ selects old content; $k_t$ is the direction of the state edit.
  \end{itemize}
  \papersource{Paper, Sec. 2 and Sec. 3.2, Eqs. (4), (7)--(8), PDF pp. 2--3.}
\end{frame}

% 5
\begin{frame}{QED adds a query term to the erase vector}
  \begin{align*}
    u_t&=b_t\odot q_t,\\
    d_t&=u_t-k_t(k_t^\top u_t)=P_{k_t}^{\perp}(b_t\odot q_t),\\
    a_t&=b_t\odot k_t+\lambda_h d_t.
  \end{align*}
  \begin{itemize}
    \item Query: selects content associated with the read.
    \item Gate: controls the correction channel by channel.
    \item Projection: enforces $d_t^\top k_t=0$ for a unit key.
  \end{itemize}
  \alert{The state update still has the form $\bar S_t+k_tr_t^\top$.}
  \papersource{Paper, Sec. 3.1, Eq. (6), PDF p. 3. Gate first, then project.}
\end{frame}

% 6
\begin{frame}{Orthogonal probe responses remain unchanged}
  For any residual $r_t$ and any probe $x$,
  \[
    \Delta S_t=k_tr_t^\top
    \quad\Longrightarrow\quad
    \Delta S_t^\top x=(x^\top k_t)r_t.
  \]
  \begin{block}{If $x\perp k_t$, the edit cannot change that response}
    This holds for both GDN-2 and QED.
  \end{block}
  With $q_t=\rho_t k_t+q_t^\perp$ and $\rho_t=q_t^\top k_t$,
  \[
    o_t^{\base}=
      \rho_t\!\left[w_t\odot v_t+
        \bar S_t^\top((\mathbf1-b_t)\odot k_t)\right]
      +\bar S_t^\top q_t^\perp.
  \]
  \papersource{Paper, Sec. 3.2, Eqs. (9)--(10), PDF p. 4. The last term bypasses direct editing.}
\end{frame}

% 7
\begin{frame}{QED routes a negative measurement through the key}
  For a shared incoming state and shared token-level quantities,
  \begin{align*}
    S_t^{\qedm}-S_t^{\base}
      &=-\lambda_h k_t(\bar S_t^\top d_t)^\top,\\[1mm]
    o_t^{\qedm}-o_t^{\base}
      &=-\lambda_h\rho_t\bar S_t^\top d_t.
  \end{align*}
  \begin{enumerate}
    \item Measure old content along $d_t$.
    \item Write its negative, scaled value along $k_t$.
    \item Reach the current read through $\rho_t=q_t^\top k_t$.
  \end{enumerate}
  If $\rho_t=0$, the current read is unchanged; later reads can still change.
  \papersource{Paper, Eqs. (11)--(12), PDF p. 4. Last sentence: direct implication of the identities.}
\end{frame}

% 8
\begin{frame}{Projection preserves a specific spectral condition}
  Let $M_t=I-k_ta_t^\top$. Its eigenvalues are
  \[
    1\;\;(K-1\text{ times}),\qquad e_t=1-a_t^\top k_t.
  \]
  Because $d_t^\top k_t=0$,
  \[
    e_t=1-\sum_i b_{t,i}k_{t,i}^2\in(0,1).
  \]
  \begin{itemize}
    \item The added correction leaves the nontrivial eigenvalue unchanged.
    \item This statement concerns the erase factor $M_t$.
    \item For $D_t=\D(\exp g_t)$, the full transition is $M_tD_t$;
      norm contraction does not follow from this spectrum alone.
  \end{itemize}
  \papersource{Paper, Secs. 3.1 and 6, PDF pp. 3, 7. Last two bullets clarify the scope by algebra.}
\end{frame}

% 9
\begin{frame}{QED learns one strength per layer and head}
  \[
    \lambda_h=\lambda_{\max}\sgm(\theta_h),\qquad
    \lambda_{\max}=0.25,\qquad \theta_h^{\rm init}=-2
  \]
  \begin{itemize}
    \item Initial strength $\approx0.0298$; one scalar per layer and head.
    \item The strength parameter uses a $10\times$ learning-rate multiplier.
    \item Same $K\times V$ state; $O(K)$ extra direction construction.
    \item The rank-one recurrence and GDN-2 kernel structure are retained.
  \end{itemize}
  \begin{block}{What the cost claim supports}
    Small additional vector work. The paper gives no measured throughput,
    latency, or memory benchmark.
  \end{block}
  \papersource{Paper, Appendix A, Eq. (13), PDF p. 9. Parameter count: 24 layers $\times$ 6 heads = 144 scalars.}
\end{frame}

% 10
\begin{frame}{Matched training isolates the recurrence change}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \begin{block}{Training}
        340M parameters\\
        24 layers, hidden size 1024\\
        6 heads, head dimension 128\\
        FineWeb-Edu, 15B tokens\\
        Llama-2 tokenizer, 2048 length\\
        Five arms, two seeds each
      \end{block}
    \end{column}
    \begin{column}{0.48\textwidth}
      \begin{block}{Evaluation}
        S-NIAH-1: 1K--32K\\
        500 examples per length\\
        4K--32K are extrapolation\\[2mm]
        FDA, SWDE, SQuAD-completion\\
        Short-document recall/control\\[2mm]
        Validation loss in nats
      \end{block}
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 4, PDF pp. 4--5. S-NIAH-2/3 are listed in setup but no scores are reported in v1.}
\end{frame}

% 11
\begin{frame}{Five arms test direction, gate, and projection}
  \centering
  \begin{tabular}{ll}
    \toprule
    Arm & Added erase term \\
    \midrule
    GDN-2 & $0$ \\
    QED-nogate-noproj & $\lambda_h q_t$ \\
    QED-nogate & $\lambda_h P_{k_t}^{\perp}q_t$ \\
    QED (full) & $\lambda_h P_{k_t}^{\perp}(b_t\odot q_t)$ \\
    QED-rand & $\lambda_h P_{k_t}^{\perp}(b_t\odot z_h)$ \\
    \bottomrule
  \end{tabular}
  \vspace{3mm}
  \begin{itemize}
    \item $z_h$: a fixed random unit direction per head.
    \item ``No gate'' removes the gate from the correction only.
    \item Projection is compared with the correction gate absent;
      the design is not a complete factorial experiment.
  \end{itemize}
  \papersource{Paper, Table 1, PDF p. 4. All arms retain the baseline term $b_t\odot k_t$.}
\end{frame}

% 12
\begin{frame}{QED improves every reported extrapolation length}
  \centering
  \includegraphics[width=\linewidth,height=0.66\textheight,keepaspectratio]{figures/fig1_sniah1.png}
  \par\medskip
  Blue vs. gray: QED beats GDN-2 at 4K, 8K, 16K, and 32K in both seeds.
  \papersource{Paper, Fig. 1, PDF p. 6. Vertical line: 2K training length. Error bars: 95\% binomial intervals.}
\end{frame}

% 13
\begin{frame}{The improvements are large, but accuracy still falls}
  \centering
  \begin{tabular}{lrrr|rrr}
    \toprule
    & \multicolumn{3}{c}{Seed 0} & \multicolumn{3}{c}{Seed 1} \\
    Length & GDN-2 & QED & $\Delta$ pp & GDN-2 & QED & $\Delta$ pp \\
    \midrule
    4K &93.4&99.6&+6.2 &79.2&96.0&+16.8\\
    8K &64.4&91.6&+27.2&50.6&84.2&+33.6\\
    16K&21.8&39.8&+18.0&21.2&59.8&+38.6\\
    32K&9.4 &15.8&+6.4 &6.0 &30.2&+24.2\\
    \bottomrule
  \end{tabular}
  \vspace{3mm}
  \begin{itemize}
    \item ``About $2\times$ usable context'' has no stated accuracy threshold.
    \item Seed 1: QED at 16K (59.8\%) exceeds baseline at 8K (50.6\%).
    \item Seed 0: the corresponding comparison is 39.8\% vs. 64.4\%.
  \end{itemize}
  \papersource{Paper, Table 3 and Sec. 6, PDF pp. 6--7. Scores: accuracy \%; gains: percentage points.}
\end{frame}

% 14
\begin{frame}{Query direction helps; gate and projection vary by seed}
  \centering
  \includegraphics[width=\linewidth,height=0.56\textheight,keepaspectratio]{figures/fig2_contrasts.png}
  \begin{itemize}
    \item Full minus random is positive at every extrapolation length.
    \item At 8K, gate effects are $+40.4$ / $-3.4$ points.
    \item At 8K, projection effects without the gate are $-39.8$ / $+70.4$ points.
  \end{itemize}
  \papersource{Paper, Fig. 2 and Table 3, PDF p. 7. Filled/open markers: seeds 0/1; original 95\% intervals.}
\end{frame}

% 15
\begin{frame}{Random directions can help, but query does better}
  \begin{block}{Seed 1 at 16K: S-NIAH-1 accuracy}
    \centering\large
    GDN-2: 21.2\% \qquad QED-rand: 47.0\% \qquad QED: 59.8\%
  \end{block}
  \begin{itemize}
    \item Random-direction control does not always return to baseline.
    \item The consistent finding is QED's advantage over this control.
    \item Gate and projection are plausible design choices, but their
      individual necessity is not established by the two runs.
  \end{itemize}
  \papersource{Paper, Table 3 and Sec. 6, PDF pp. 6--7. Interpretation follows the observed contrasts.}
\end{frame}

% 16
\begin{frame}{Document recall improves selectively}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.54\textwidth}
      \includegraphics[width=\linewidth,height=0.72\textheight,keepaspectratio]{figures/fig3_recall.png}
    \end{column}
    \begin{column}{0.43\textwidth}
      \textbf{Full QED minus GDN-2}\\[2mm]
      FDA: $+3.35$ / $+3.73$ pp\\
      Final accuracy: 7.80\% / 7.99\%\\[3mm]
      SQuAD-completion:\\
      $-0.10$ / $-0.16$ pp\\[3mm]
      SWDE: $+2.43$ / $0.00$ pp\\[3mm]
      Short documents test recall,\\
      not context extrapolation.
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 5.4, Table 3 and Fig. 3, PDF pp. 6--8. Paired numbers: seeds 0/1.}
\end{frame}

% 17
\begin{frame}{Language modeling loss is nearly unchanged}
  \centering
  \begin{tabular}{lrrr}
    \toprule
    Seed & GDN-2 & QED & QED minus GDN-2 \\
    \midrule
    0 &2.3714&2.3719&$+0.0005$\\
    1 &2.3746&2.3735&$-0.0011$\\
    \bottomrule
  \end{tabular}
  \vspace{5mm}
  \begin{itemize}
    \item Validation loss in nats; lower is better.
    \item Across all five arms and both seeds, the total spread is 0.0034 nats.
    \item Together with the recall results, this supports an improvement
      in access to compressed memory.
  \end{itemize}
  \papersource{Paper, Table 2, PDF p. 5; discussion, Sec. 6, PDF p. 6.}
\end{frame}

% 18
\begin{frame}{Inference strength affects the two seeds differently}
  \[
    \lambda_h^{(s)}=s\lambda_h,\qquad \text{all trained weights fixed}
  \]
  \centering
  \begin{tabular}{lrrrrrr}
    \toprule
    S-NIAH-1 at 16K & $-1$ & $0$ & $0.5$ & $1$ & $1.5$ & $2$\\
    \midrule
    Seed 0 &39.8&41.4&42.0&41.8&41.6&43.6\\
    Seed 1 &44.8&55.0&61.0&65.6&69.8&73.0\\
    \bottomrule
  \end{tabular}
  \vspace{3mm}
  \begin{itemize}
    \item Seed 1: larger correction helps at all four extrapolation lengths.
    \item Seed 0: small changes with no consistent direction.
    \item Scale is applied throughout prefill and decoding, changing the trajectory.
  \end{itemize}
  \papersource{Paper, Appendix A.1, Table 4, PDF pp. 9--10. Scores: accuracy \%; $s=1$ is the trained setting.}
\end{frame}

% 19
\begin{frame}{The intervention has a narrower interpretation}
  \begin{itemize}
    \item $s=0$ removes QED from a QED-trained model; it is not the independently
      trained GDN-2 baseline.
    \item The incoming state also depends on $s$, so this is not just a
      final-read intervention.
    \item Table 4's $s=1$ differs from Table 3: seed 1 at 16K is
      65.6\% vs. 59.8\%. The paper does not explain the discrepancy.
    \item A gain at $s=2$ motivates a training-time strength sweep; it does
      not establish that $\lambda_{\max}=0.25$ constrained training.
  \end{itemize}
  \papersource{Paper, Tables 3--4, PDF pp. 7, 10; Appendix A.1, PDF p. 9. Compare strengths within Table 4.}
\end{frame}

% 20
\begin{frame}{The contribution is query-aware state maintenance}
  \begin{enumerate}
    \item \textbf{Mechanism:} measure old content using the query; write a
      correction along the key, keeping a rank-one edit.
    \item \textbf{Evidence:} better S-NIAH-1 extrapolation and FDA recall in
      two matched 340M runs, with nearly unchanged LM loss.
    \item \textbf{Open issue:} component effects and inference-time reliance
      vary across seeds.
  \end{enumerate}
  \begin{block}{Useful next experiments}
    More seeds; training-time $\lambda_{\max}$ sweeps; gated/no-projection
    control; larger scales and measured kernel overhead.
  \end{block}
  \papersource{Paper, Secs. 6--7, PDF pp. 6--8. The missing control and kernel benchmark are discussion suggestions.}
\end{frame}

% 21
\begin{frame}{References and paper status}
  \textbf{Primary reading}\\[1mm]
  D. Gupta, A. Das, D. Gupta.\\
  \emph{The Query Knows What to Forget: A Second Erase Direction for Linear Attention.}\\
  \href{https://arxiv.org/abs/2608.13668v1}{arXiv:2608.13668v1}, 2026.\\[4mm]
  \textbf{Status checked 2026-10-30}\\[1mm]
  Formal publication not found; public code not found.\\
  \href{https://arxiv.org/abs/2608.13668}{Current arXiv record}.\\[4mm]
  \textbf{Task background}\\[1mm]
  S. Arora et al., \emph{Simple linear attention language models balance the
  recall-throughput tradeoff.}\\
  \href{https://arxiv.org/abs/2402.18668v2}{arXiv:2402.18668v2}: FDA, SWDE, and SQuAD task definitions.
  \papersource{All experimental numbers in this deck are from QED v1. Original figure numbers are retained.}
\end{frame}

\appendix

% 22
\begin{frame}{Backup: the progression to GDN-2}
  \centering
  \begin{tabular}{lll}
    \toprule
    Model & Decay & Erase / write \\
    \midrule
    DeltaNet & None & Shared scalar $\beta_t$ \\
    GDN & Scalar $\alpha_t$ & Shared scalar $\beta_t$ \\
    KDA & Channel-wise & Shared scalar $\beta_t$ \\
    GDN-2 & Channel-wise & Separate $b_t$ and $w_t$ \\
    QED & Channel-wise & Query term in erase vector \\
    \bottomrule
  \end{tabular}
  \vspace{4mm}
  \[
    S_t^{\rm DeltaNet}=(I-\beta_tk_tk_t^\top)S_{t-1}
                       +\beta_tk_tv_t^\top
  \]
  \begin{itemize}
    \item GDN-2 is the directly trained baseline in this study.
    \item EDA is discussed as prior work with separate erase/write addresses.
  \end{itemize}
  \papersource{Paper, Sec. 2, Eqs. (1)--(4), PDF pp. 2--3.}
\end{frame}

% 23
\begin{frame}{Backup: a two-dimensional example}
  \[
    k=\begin{pmatrix}1\\0\end{pmatrix},\quad
    q=\begin{pmatrix}0.6\\0.8\end{pmatrix},\quad
    b=\begin{pmatrix}0.5\\0.5\end{pmatrix},\quad
    \bar S=\begin{pmatrix}2\\3\end{pmatrix},\quad
    wv=1,\quad\lambda=0.2
  \]
  \begin{align*}
    r^{\base}&=1-\bar S^\top(b\odot k)=0,
      &o^{\base}&=3.6,\\
    d&=(0,0.4)^\top,
      &\bar S^\top d&=1.2,\\
    \Delta S&=(-0.24,0)^\top,
      &\Delta o&=-0.144.
  \end{align*}
  \begin{block}{The second state coordinate stays at 3}
    Its content determines a correction to the first coordinate.
    The new output is 3.456.
  \end{block}
  \papersource{Teaching example derived from Eqs. (6), (11)--(12); not an experiment from the paper.}
\end{frame}

% 24
\begin{frame}{Backup: eigenvalue preservation is not norm contraction}
  Choose $k=(1,0)^\top$, $q=(0,1)^\top$, $b=(0.5,0.5)^\top$,
  and $\lambda=0.2$. Then
  \[
    a=(0.5,0.1)^\top,\qquad
    M=I-ka^\top=\begin{pmatrix}0.5&-0.1\\0&1\end{pmatrix}.
  \]
  \[
    \operatorname{spec}(M)=\{0.5,1\},\qquad
    \left\|M\begin{pmatrix}0\\1\end{pmatrix}\right\|_2
      =\sqrt{1.01}>1.
  \]
  The projection preserves $1-a^\top k$, but a non-normal erase factor
  can amplify some vectors. Channel-wise decay adds another matrix factor.
  \papersource{Independent algebraic clarification of the claim in Sec. 3.1; not a measured model instability.}
\end{frame}

% 25
\begin{frame}{Backup: interpreting the projection diagnostic}
  \begin{itemize}
    \item Without projection, the paper gives the possible range
      $e=1-a^\top k\in[-0.25,1.25]$.
    \item In 1.18M token-head samples from two unprojected models,
      the observed range is $[-0.011,0.985]$.
    \item The paper also says counterfactual projection changes $e$ by
      less than $10^{-3}$, without specifying the statistic.
  \end{itemize}
  \begin{block}{A numerical point to clarify}
    A uniform per-sample change below $10^{-3}$ would conflict with a
    minimum of $-0.011$ and projected $e\in(0,1)$. An average change would
    not imply that uniform bound.
  \end{block}
  \papersource{Paper, Sec. 6, PDF p. 7. The consistency observation is this reading's analysis.}
\end{frame}

% 26
\begin{frame}{Backup: optimization and intervention details}
  \begin{itemize}
    \item AdamW, learning rate $3\times10^{-4}$; WSD decay begins at 13.5B
      of 15B training tokens.
    \item Global batch: 524,288 tokens; BF16 with FP32 master weights.
    \item RMSNorm, SwiGLU, tied embeddings.
    \item Strength sweep: same 500 deterministic examples per length
      for every $s$, with an exact McNemar test against $s=1$ specified.
    \item Paired outcome counts and p-values are not reported; marginal
      accuracies alone cannot reconstruct that test.
  \end{itemize}
  \papersource{Paper, Sec. 4.2 and Appendix A.1, PDF pp. 5, 9--10.}
\end{frame}

\end{document}