% Based on arxiv-paper-reader/assets/beamer-template.tex.
% Independent reading presentation; all experimental values are from the paper.
% Main talk: about 25 minutes. Backup starts after the references frame.
% Build: python3 build_slides.py slides.en.tex
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\pdfstringdefDisableCommands{\def\\{ }\def\translate#1{#1}}

\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\definecolor{Muted}{RGB}{94,108,119}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hspace{8mm}{\color{Muted}\scriptsize MDN \enspace|\enspace Paper discussion}
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{Muted}#1\par}}
\newcommand{\Smat}{\mathbf S}
\newcommand{\Mmat}{\mathbf M}
\newcommand{\kk}{\boldsymbol k}
\newcommand{\vv}{\boldsymbol v}
\newcommand{\qq}{\boldsymbol q}
\newcommand{\pp}{\boldsymbol p}
\newcommand{\ve}{\widetilde{\boldsymbol v}}
\newcommand{\Vres}{\widetilde{\mathbf V}}
\newcommand{\Khat}{\widehat{\mathbf K}}

\title[MDN]{MDN: Parallelizing Stepwise Momentum\\for Delta Linear Attention}
\subtitle{Exact chunkwise computation for a two-state memory}
\author{\small Paper by Yulong Huang, Xiang Liu, Hongxiang Huang, Xiaopeng Lin,\\
Zunchang Liu, Xiaowen Chu, Zeke Xie, and Bojun Cheng}
\institute{The Hong Kong University of Science and Technology (Guangzhou)\\[2mm]
\textit{Independent paper reading presentation}}
\date{\href{https://arxiv.org/abs/2605.05838v1}{arXiv:2605.05838v1} \enspace|\enspace 20 October 2026}

\begin{document}

\begin{frame}
  \titlepage
  \vspace{-4mm}
  \begin{center}
  \alert{Preserve token-by-token error feedback while training in chunks.}
  \end{center}
\end{frame}

\begin{frame}{The gap: historical error feedback with efficient training}
  \begin{itemize}
    \item Delta attention corrects a compact memory using the current error.
    \item Momentum keeps earlier error directions active in future updates.
    \item The challenge is parallelizing the resulting causal dependencies.
  \end{itemize}
  \medskip
  \centering
  \includegraphics[width=.96\linewidth,height=3.1cm,keepaspectratio]{figures/fig-01-causality.pdf}
  \papersource{Paper, Sec. 1 and Fig. 1, PDF pp. 1--2. The figure compares the authors' selected update schemes.}
\end{frame}

\begin{frame}{MDN adds a momentum matrix to the delta update}
  \[
    \mathcal L(\widetilde{\Smat}_{t-1})
    =\tfrac12\|\vv_t-\widetilde{\Smat}_{t-1}^{\top}\kk_t\|_2^2,
    \qquad \widetilde{\Smat}_{t-1}=\alpha_t\Smat_{t-1}
  \]
  \begin{block}{One token, one causal memory update}
  \[
  \begin{aligned}
    \ve_t&=\vv_t-\alpha_t\Smat_{t-1}^{\top}\kk_t,\\
    \Mmat_t&=\mu_t\Mmat_{t-1}-\eta_t\kk_t\ve_t^\top,\\
    \Smat_t&=\alpha_t\Smat_{t-1}-\beta_t\Mmat_t,
    \qquad \boldsymbol o_t=\Smat_t^\top\qq_t.
  \end{aligned}
  \]
  \end{block}
  \begin{itemize}
    \item $\Smat_t,\Mmat_t\in\mathbb R^{d_k\times d_v}$: memory and momentum.
    \item $\alpha$: memory retention; $\mu$: momentum retention.
    \item $\beta$: update step; $\eta$: current gradient scale.
  \end{itemize}
  \papersource{Paper, Sec. 3.1, Eqs. (3)--(5), PDF p. 4. Setting $\mu=0,\eta=1$ recovers GDN.}
\end{frame}

\begin{frame}{One error can affect several future writes}
  Set $\widehat{\kk}_i=\eta_i\kk_i$ and condition on the residual sequence.
  \medskip
  \begin{block}{Contribution of the first residual to the second state}
  \[
  \underbrace{
  \left(\alpha_2\beta_1+\beta_2\mu_2\right)
  }_{\text{two propagation paths}}
  \widehat{\kk}_1\ve_1^\top
  \]
  \end{block}
  \begin{columns}[T]
    \begin{column}{.47\textwidth}
      \textbf{Through memory}\\[2mm]
      $\alpha_2\beta_1$ retains the write made at the first step.
    \end{column}
    \begin{column}{.47\textwidth}
      \textbf{Through momentum}\\[2mm]
      $\beta_2\mu_2$ writes the retained error direction again.
    \end{column}
  \end{columns}
  \medskip
  Actual residuals are recomputed from the preceding memory at each step.
  \papersource{Teaching derivation from Eqs. (4)--(5), PDF p. 4. This is inner memory optimization; outer training uses AdamW.}
\end{frame}

\begin{frame}{The memory sits inside a familiar language-model block}
  \begin{columns}[c]
    \begin{column}{.57\textwidth}
      \includegraphics[width=\linewidth,height=5.9cm,keepaspectratio]{figures/fig-03-architecture.pdf}
    \end{column}
    \begin{column}{.40\textwidth}
      \begin{itemize}
        \item Project, short-convolve, and activate $q,k,v$.
        \item Normalize queries and keys.
        \item Generate four scalar gates per head.
        \item Update memory and momentum.
        \item RMSNorm, output gate, projection.
      \end{itemize}
      \medskip
      Comba-style query correction is also retained.
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 3.3 and Fig. 3, PDF pp. 6--7. Red components identify the momentum-related additions.}
\end{frame}

\begin{frame}{Reorder the triangular sum to isolate scalar coefficients}
  Within one chunk, write $\widehat{\kk}_j=\eta_j\kk_j$.
  \[
  \Smat_i=\bar\alpha_i\Smat_0-b_i\Mmat_0
  +\sum_{j\le i}\gamma_{i,j}\widehat{\kk}_j\ve_j^\top
  \]
  \[
  \begin{aligned}
    \bar\alpha_i&=\prod_{r\le i}\alpha_r,\quad
    \bar\mu_i=\prod_{r\le i}\mu_r,\quad
    c_i=\sum_{r\le i}\frac{\beta_r\bar\mu_r}{\bar\alpha_r},\\
    b_i&=\bar\alpha_i c_i,\qquad
    \gamma_{i,j}=\frac{\bar\alpha_i}{\bar\mu_j}(c_i-c_{j-1}).
  \end{aligned}
  \]
  \begin{block}{Why the reordering helps}
  Fix the error's origin $j$ and aggregate all later writes into $\gamma_{i,j}$.
  The traversal order changes; the causal domain $j\le i$ is preserved.
  \end{block}
  \papersource{Paper, Sec. 3.1, Eqs. (7)--(9), p. 4; Appendix C, Eqs. (30)--(31), p. 17. We use the consistent Appendix C expression.}
\end{frame}

\begin{frame}{Solve the unknown residuals with a lower triangular system}
  Define $\pp_i=\alpha_i\kk_i$ and stack token vectors as matrix rows.
  \[
  \Gamma^-_{i,j}=
  \begin{cases}
    \gamma_{i-1,j},&j<i,\\
    0,&j\ge i .
  \end{cases}
  \quad
  \mathbf T=\left[\mathbf I+
    (\mathbf P\Khat^\top)\odot\boldsymbol{\Gamma}^-\right]^{-1}
  \]
  \[
  \begin{aligned}
    \mathbf U&=\mathbf T\mathbf V,\\
    \mathbf Y&=\mathbf T\operatorname{Diag}(\bar{\boldsymbol\alpha}^{-})\mathbf P,\qquad
    \mathbf Z=\mathbf T\operatorname{Diag}(\boldsymbol b^{-})\mathbf P,\\
    \Vres&=\mathbf U-\mathbf Y\Smat_0+\mathbf Z\Mmat_0.
  \end{aligned}
  \]
  \begin{itemize}
    \item $\mathbf U,\mathbf Y,\mathbf Z$ depend only on chunk-local inputs.
    \item $\Gamma^-$ is $\Gamma$ shifted down one row: the error reads $\Smat_{i-1}$.
    \item $\bar{\boldsymbol\alpha}^{-}=(1,\bar\alpha_1,\ldots)$; $\boldsymbol b^{-}=(0,b_1,\ldots)$.
  \end{itemize}
  \papersource{Paper, Eqs. (15)--(18), p. 5; Appendix C, p. 18; row shift confirmed by Appendix E, p. 22.}
\end{frame}

\begin{frame}{Chunk outputs combine incoming states and local writes}
  \[
  \begin{aligned}
    \mathbf O={}&
    \underbrace{\operatorname{Diag}(\bar{\boldsymbol\alpha})\mathbf Q\Smat_0
    -\operatorname{Diag}(\boldsymbol b)\mathbf Q\Mmat_0}_{\text{history entering the chunk}}\\
    &+\underbrace{[(\mathbf Q\Khat^\top)\odot\boldsymbol{\Gamma}]\Vres}_{\text{writes inside the chunk}}.
  \end{aligned}
  \]
  \medskip
  \begin{itemize}
    \item Matrix operations process the chunk; two final states pass to the next chunk.
    \item Decoding uses the same token recurrence with fixed-size $\Smat,\Mmat$.
    \item Chunking is an exact algebraic reformulation of the recurrence.
    \item At fixed head dimensions, the recurrent matrix state has twice as many elements as a single-state rule.
  \end{itemize}
  \papersource{Paper, Sec. 3.1, PDF p. 5; Appendix C, Eqs. (40)--(42), PDF p. 18. Exactness refers to real arithmetic.}
\end{frame}

\begin{frame}{The implementation trades saved residuals for throughput}
  \begin{columns}[T]
    \begin{column}{.47\textwidth}
      \begin{block}{Coefficient computation}
      \begin{itemize}
        \item Prefix sums of $\log\alpha,\log\mu$.
        \item Stable prefix log-sum-exp for $c_i$.
        \item Local $C\times C$ coefficient matrices.
      \end{itemize}
      \end{block}
      Pseudocode uses $C=64$ by default.
    \end{column}
    \begin{column}{.47\textwidth}
      \begin{block}{Backward computation}
      \begin{itemize}
        \item Save corrected values $\Vres$.
        \item Reconstruct states for gradients.
        \item Avoid storing every full state pair.
      \end{itemize}
      \end{block}
      Additional activations buy training speed.
    \end{column}
  \end{columns}
  \medskip
  Fixed chunk size gives linear scaling in sequence length.
  The triangular broadcast still performs $O(C^2)$ coefficient work.
  \papersource{Paper, Sec. 3.1, p. 5; Algorithm 1, p. 19; Appendix E, pp. 20--22. The paper's constant-time claim treats $C$ as fixed.}
\end{frame}

\begin{frame}{Momentum permits damped oscillatory modes}
  \centering
  \includegraphics[width=.96\linewidth,height=3.4cm,keepaspectratio]{figures/fig-02-spectrum.pdf}
  \[
  r_t=\alpha_t+\mu_t-\alpha_t\beta_t\eta_t,\quad
  \lambda_\pm=\frac{r_t\pm\sqrt{r_t^2-4\alpha_t\mu_t}}2
  \]
  \begin{itemize}
    \item For normalized keys, a negative discriminant gives a complex pair.
    \item Its modulus is $\sqrt{\alpha_t\mu_t}$: decay with phase evolution.
    \item These are frozen-step spectra, not a bound on all time-varying products.
  \end{itemize}
  \papersource{Paper, Fig. 2, p. 6; Appendix F, Eq. (60), p. 23. $r_t$ denotes the trace to avoid overloading temperature $\tau$.}
\end{frame}

\begin{frame}{Coupled gates restrict the spectral region}
  \[
  \eta_t=1+\tanh(\mathbf W_\eta\boldsymbol x_t/\tau),\qquad
  \theta_t=\arctan(s\eta_t)
  \]
  \[
  \alpha_{\max}=\cos^2\theta_t,\qquad
  \beta_{\max}=\sin^2\theta_t
  \quad\Longrightarrow\quad \alpha_t+\beta_t\le1
  \]
  \[
  \begin{aligned}
    \log\alpha_t&=f(\mathbf W_\alpha\boldsymbol x_t)+\log\alpha_{\max},\\
    \beta_t&=\sigma(\mathbf W_\beta\boldsymbol x_t)\beta_{\max},\\
    \log\mu_t&=\max\{f(\mathbf W_\mu\boldsymbol x_t),\mu_{\min}^{\log}\}.
  \end{aligned}
  \]
  \begin{itemize}
    \item $f(z)=-a\,\operatorname{softplus}(z+b)$; $\eta_t\in(0,2)$.
    \item Text/illustration use a $-1$ momentum log floor.
    \item \alert{Reported experiments use $-2$}, as specified in Tables 4 and 6.
  \end{itemize}
  \papersource{Paper, Sec. 3.3, pp. 6--7; Tables 4 and 6, pp. 9 and 25.}
\end{frame}

\begin{frame}{Evaluation separates model quality, recall, and cost}
  \centering
  \begin{tabular}{lcc}
  \toprule
   &400M&1.3B\\
  \midrule
  Training tokens&15B&100B\\
  Tokens per batch&0.5M&1M\\
  Training context&4K&4K\\
  Warmup tokens&0.5B&1B\\
  \bottomrule
  \end{tabular}
  \medskip
  \begin{itemize}
    \item SlimPajama subset, GPT-2 tokenizer, AdamW; matched data/tokens.
    \item Transformer, Mamba2, GDN, Comba, and pure-linear KDA baselines.
    \item PPL; eight-task reasoning mean; six-task retrieval mean at 2K.
    \item MQAR, LongBench at 16K, RULER NIAH, and H100 efficiency.
  \end{itemize}
  \papersource{Paper, Sec. 4, pp. 7--9; Appendix G, pp. 23--24. Matched parameter scale does not imply matched recurrent state capacity.}
\end{frame}

\begin{frame}{At 400M, recall improves more than reasoning}
  \centering
  \begin{tabular}{lrrrr}
  \toprule
  Model&Lamb. PPL $\downarrow$&Wiki. PPL $\downarrow$&Reasoning $\uparrow$&Retrieval $\uparrow$\\
  \midrule
  Transformer&54.36&32.80&48.06&30.42\\
  Mamba2&60.42&33.45&47.85&21.12\\
  GDN&45.63&32.10&48.51&22.93\\
  Comba&46.19&31.73&48.91&24.01\\
  KDA&43.44&31.96&49.06&24.47\\
  \textbf{MDN}&\textbf{41.62}&\textbf{31.51}&\textbf{49.42}&\textbf{26.76}\\
  \bottomrule
  \end{tabular}
  \medskip
  \begin{itemize}
    \item Versus the strongest other linear model: $+0.36$ reasoning points, $+2.29$ retrieval points.
    \item Transformer retains the best retrieval mean: $30.42$.
  \end{itemize}
  \papersource{Paper, Table 2, PDF p. 8. 400M parameters, 15B training tokens; retrieval uses 2K inputs. Reasoning averages mixed acc/acc\_n.}
\end{frame}

\begin{frame}{The pattern persists at 1.3B}
  \centering
  \begin{tabular}{lrrrr}
  \toprule
  Model&Lamb. PPL $\downarrow$&Wiki. PPL $\downarrow$&Reasoning $\uparrow$&Retrieval $\uparrow$\\
  \midrule
  Transformer&17.90&18.99&57.95&40.60\\
  Mamba2&18.20&19.14&57.10&32.45\\
  GDN&16.12&18.51&57.44&32.80\\
  Comba&15.17&18.37&58.49&35.25\\
  KDA&16.83&19.24&58.56&32.62\\
  \textbf{MDN}&\textbf{14.87}&\textbf{18.03}&\textbf{58.82}&\textbf{36.14}\\
  \bottomrule
  \end{tabular}
  \medskip
  \begin{itemize}
    \item Best PPL and reasoning mean in this comparison.
    \item Retrieval: $+0.89$ points versus Comba, $-4.46$ versus Transformer.
    \item Gains vary by task: MDN's DROP score is below GDN and Comba.
  \end{itemize}
  \papersource{Paper, Table 2, PDF p. 8. 1.3B parameters, 100B training tokens. Comparisons use this paper's matched evaluation protocol.}
\end{frame}

\begin{frame}{MQAR gains emerge at high association density}
  \centering
  \includegraphics[width=.97\linewidth,height=4.6cm,keepaspectratio]{figures/fig-04-mqar.pdf}
  \begin{itemize}
    \item Train on 64--256 tokens with 4--64 key--value pairs.
    \item Test up to 2,048 tokens and 512 pairs; MDN degrades later.
    \item Both length and pair count change; KDA initialization also matters.
  \end{itemize}
  \papersource{Paper, Fig. 4, p. 7; Table 5, p. 24. Panels: embedding dimension 128/256; y-axis: accuracy.}
\end{frame}

\begin{frame}{LongBench gains are strongest in code and summarization}
  \centering
  \begin{tabular}{lrrrr}
  \toprule
  Score $\uparrow$&GDN&Comba&KDA&MDN\\
  \midrule
  Overall mean&19.28&18.11&18.62&\textbf{20.18}\\
  LCC (code)&42.94&44.74&37.99&\textbf{50.50}\\
  RepoBench-P&30.70&36.60&33.39&\textbf{39.13}\\
  GovReport&7.81&9.60&10.02&\textbf{10.24}\\
  QMSum&15.98&16.52&15.59&\textbf{17.20}\\
  MultiNews&16.82&16.74&16.95&\textbf{18.85}\\
  \bottomrule
  \end{tabular}
  \medskip
  \begin{itemize}
    \item 1.3B models: train at 4K, evaluate at 16K.
    \item Overall: $+0.90$ score points over GDN.
    \item Task trade-off: SSM is $15.65$ for MDN versus $26.27$ for GDN.
  \end{itemize}
  \papersource{Paper, Table 3, PDF p. 8. Overall mean covers all 14 tasks, not only the rows shown. Task scores use different metrics.}
\end{frame}

\begin{frame}{NIAH improves beyond the 4K training length}
  \begin{columns}[c]
    \begin{column}{.66\textwidth}
      \includegraphics[width=\linewidth,height=5.7cm,keepaspectratio]{figures/fig-05-niah.pdf}
    \end{column}
    \begin{column}{.31\textwidth}
      \textbf{At 8K context}\\[2mm]
      \begin{tabular}{lrr}
      \toprule
      &MDN&Gain\\
      \midrule
      MK&38.60&13.40\\
      MQ&35.15&11.45\\
      MV&27.60&8.95\\
      \bottomrule
      \end{tabular}
      \medskip

      Gain: points over the strongest other baseline per task.
      \medskip

      Multi-needle accuracy still falls at 16K.
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 5 and accompanying text, PDF p. 8. Dashed vertical line: 4K training context; red line: MDN.}
\end{frame}

\begin{frame}{Decoding remains close to recurrent baselines}
  \begin{columns}[c]
    \begin{column}{.59\textwidth}
      \includegraphics[width=\linewidth,height=5.7cm,keepaspectratio]{figures/fig-06a-decoding.pdf}
    \end{column}
    \begin{column}{.38\textwidth}
      \begin{itemize}
        \item MDN tracks GDN, Comba, and KDA as generated length grows.
        \item Transformer time grows faster at long lengths.
        \item Both recurrent matrices remain fixed in size.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 6 (left panel), PDF p. 8. 1.3B models, one H100. The plotted time is not per-token latency.}
\end{frame}

\begin{frame}{Training speed improves when residuals are saved}
  \begin{columns}[c]
    \begin{column}{.64\textwidth}
      \includegraphics[width=\linewidth,height=5.7cm,keepaspectratio]{figures/fig-06b-training.pdf}
    \end{column}
    \begin{column}{.33\textwidth}
      \begin{itemize}
        \item Comparable to KDA and Mamba2.
        \item Below GDN and Comba.
        \item Red solid line saves $\Vres$; red dashed line does not.
      \end{itemize}
      \medskip
      Reported memory:
      \[
      41.25\to44.89\ {\rm GB}.
      \]
    \end{column}
  \end{columns}
  \papersource{Paper, Fig. 6 (right panel), PDF p. 8. 1.3B, one H100; sequence length $\times$ batch $=16{,}384$ tokens.}
\end{frame}

\begin{frame}{The momentum ablation loses 6.64 retrieval points}
  \centering
  \begin{tabular}{lrrr}
  \toprule
  400M variant&Lamb. PPL $\downarrow$&Reasoning $\uparrow$&Retrieval $\uparrow$\\
  \midrule
  MDN&41.62&49.42&26.76\\
  Without output correction&42.31&49.19&25.52\\
  Without momentum$^\ast$&47.01&49.26&20.12\\
  Without $\beta_{\max}$&42.72&48.93&26.40\\
  $\eta=2\sigma(\cdot)$&49.10&49.41&25.54\\
  \bottomrule
  \end{tabular}
  \medskip
  \begin{itemize}
    \item $^\ast$Runnable proxy: GDN with $\alpha_{\max},\beta_{\max}$ constraints.
    \item Directly setting $\mu=0$ breaks this parallel kernel's divisions.
    \item No momentum floor: NaN at step 70; no $\alpha_{\max}$: NaN at step 1.
  \end{itemize}
  \papersource{Paper, Table 4, p. 9; Table 6 and footnotes, p. 25. The mathematical recurrence itself permits $\mu=0$.}
\end{frame}

\begin{frame}{The momentum floor changes the quality trade-off}
  \centering
  \begin{tabular}{lrrrr}
  \toprule
  Minimum $\log\mu$&Lamb. PPL $\downarrow$&Wiki. PPL $\downarrow$&Reasoning $\uparrow$&Retrieval $\uparrow$\\
  \midrule
  $-2$ (reported)&\textbf{41.62}&31.51&49.42&\textbf{26.76}\\
  $-1.5$&42.65&31.50&48.94&26.03\\
  $-1.357$&44.90&31.44&49.50&25.31\\
  $-1$&43.53&\textbf{31.39}&\textbf{49.88}&25.85\\
  \bottomrule
  \end{tabular}
  \medskip
  \begin{itemize}
    \item A larger floor forces more momentum retention.
    \item $-2$ gives the strongest retrieval and LAMBADA PPL.
    \item $-1$ improves the reasoning mean and WikiText PPL.
    \item There is no single monotonic improvement across metrics.
  \end{itemize}
  \papersource{Paper, Table 4, PDF p. 9. All rows are 400M models under the same training setup.}
\end{frame}

\begin{frame}{Sparse full-attention layers are promising, with trade-offs}
  \centering
  \begin{tabular}{lrrrr}
  \toprule
  400M hybrid&Lamb. PPL $\downarrow$&Wiki. PPL $\downarrow$&Reasoning $\uparrow$&Retrieval $\uparrow$\\
  \midrule
  Comba-H 3:1&\textbf{40.88}&29.92&49.35&\textbf{34.72}\\
  KDA-H 3:1&41.78&\textbf{29.49}&48.93&34.46\\
  MDN-H 3:1&46.71&30.09&48.61&33.84\\
  MDN-H 7:1&42.95&30.32&\textbf{49.68}&34.37\\
  \bottomrule
  \end{tabular}
  \medskip
  \begin{itemize}
    \item Ratios count linear layers : full-attention layers.
    \item MDN's 7:1 improves reasoning and retrieval over its 3:1 variant.
    \item At 3:1, MDN does not dominate Comba or KDA.
    \item This table reports quality; it does not measure the speed gain of 7:1.
  \end{itemize}
  \papersource{Paper, Table 4, PDF p. 9. Selected hybrid rows; all use the 400M training budget.}
\end{frame}

\begin{frame}{Memory updates are more active during decoding}
  \centering
  \includegraphics[width=.95\linewidth,height=3.3cm,keepaspectratio]{figures/fig-07-state-changes.pdf}
  \[
  \Delta S_t=\frac1n\sum_{i=1}^n
  \left\|\Smat_t^{(i)}-\Smat_{t-1}^{(i)}\right\|_F
  \]
  \begin{itemize}
    \item MDN has larger average step-to-step state changes.
    \item The authors connect this richer dynamics to retrieval improvements.
    \item Change magnitude is a descriptive probe, not a measure of information quality.
  \end{itemize}
  \papersource{Paper, Fig. 7 and Sec. 4.3, PDF p. 9. The paper does not define the shaded-band statistic.}
\end{frame}

\begin{frame}{The contribution is an exact, practical momentum memory}
  \begin{enumerate}
    \item \textbf{Algorithm:} geometric reordering and triangular residual solving preserve token-level updates.
    \medskip
    \item \textbf{Quality:} recall and long-context gains over strong linear baselines; full attention still leads 2K retrieval.
    \medskip
    \item \textbf{Cost:} an extra recurrent state and saved residuals; training trails GDN/Comba.
  \end{enumerate}
  \medskip
  \begin{block}{What remains open}
  Larger than 1.3B; faster dual-state kernels; broader hybrid configurations;
  stability of input-dependent matrix products.
  \end{block}
  \papersource{Paper, Sec. 5, p. 9; Appendix I, p. 24. Matrix-product stability is a reading inference beyond the frozen-step analysis.}
\end{frame}

\begin{frame}{Reference and source conventions}
  \textbf{Yulong Huang, Xiang Liu, Hongxiang Huang, Xiaopeng Lin,\\
  Zunchang Liu, Xiaowen Chu, Zeke Xie, and Bojun Cheng.}
  \medskip

  \textit{MDN: Parallelizing Stepwise Momentum for Delta Linear Attention.}
  \medskip

  \href{https://arxiv.org/abs/2605.05838v1}{arXiv:2605.05838v1}, submitted 7 May 2026.
  \medskip

  Author repository: \href{https://github.com/HuuYuLong/MomentumDeltaNet}{github.com/HuuYuLong/MomentumDeltaNet}
  \medskip
  \begin{itemize}
    \item All experimental numbers and figures are from this paper version.
    \item PDF page numbers are one-based and match the printed page numbers.
    \item Teaching derivations and interpretive limits are labeled explicitly.
    \item Backup slides clarify formula inconsistencies that affect implementation.
  \end{itemize}
\end{frame}

\appendix

\begin{frame}{Backup: a one-row shift is essential for causality}
  \[
  \boldsymbol{\Gamma}=
  \begin{pmatrix}
    \gamma_{1,1}&0&0\\
    \gamma_{2,1}&\gamma_{2,2}&0\\
    \gamma_{3,1}&\gamma_{3,2}&\gamma_{3,3}
  \end{pmatrix},
  \qquad
  \boldsymbol{\Gamma}^-=
  \begin{pmatrix}
    0&0&0\\
    \gamma_{1,1}&0&0\\
    \gamma_{2,1}&\gamma_{2,2}&0
  \end{pmatrix}.
  \]
  \begin{block}{The residual at position $i$ reads $\Smat_{i-1}$}
  \[
    \ve_i^\top=\vv_i^\top-\pp_i^\top\Smat_{i-1}.
  \]
  Its earlier-write coefficients are $\gamma_{i-1,j}$, not $\gamma_{i,j}$.
  \end{block}
  Simply deleting the diagonal from $\Gamma$ produces the wrong coefficients.
  \papersource{Paper, Appendix C, PDF p. 18; Appendix E, PDF p. 22, code lines 130--133. Main-text p. 5 wording is imprecise.}
\end{frame}

\begin{frame}{Backup: use the consistent log-domain coefficients}
  \[
  A_i=\sum_{r\le i}\log\alpha_r,\quad
  U_i=\sum_{r\le i}\log\mu_r,\quad
  \ell_i=\operatorname{logcumsumexp}_{r\le i}
  (\log\beta_r+U_r-A_r).
  \]
  \[
  b_i=\exp(A_i+\ell_i),\qquad
  \gamma_{i,j}=
  \exp(A_i+\ell_i-U_j)
  [1-\exp(\ell_{j-1}-\ell_i)].
  \]
  \begin{itemize}
    \item $\ell_0=-\infty$; coefficients are defined only on the causal region.
    \item Eq. (8): use $\operatorname{Diag}(\Gamma_{t,:})$, as in Appendix C.
    \item Eq. (10): $b_i=\bar\alpha_i c_i$ requires cumulative $\log\alpha$.
    \item Appendix E's \texttt{b\_t} implements this consistent definition.
  \end{itemize}
  \papersource{Paper, Eqs. (8)--(10), pp. 4--5; Eq. (30), p. 17; Appendix D, p. 19; Appendix E, p. 21.}
\end{frame}

\begin{frame}{Backup: the full spectrum includes orthogonal directions}
  \[
  \mathbf A_t=
  \begin{pmatrix}
    \alpha_t\mathbf I-\alpha_t\beta_t\eta_t\kk_t\kk_t^\top&
    -\beta_t\mu_t\mathbf I\\
    \alpha_t\eta_t\kk_t\kk_t^\top&\mu_t\mathbf I
  \end{pmatrix}
  \]
  \[
  \operatorname{Spec}(\mathbf A_t)=
  \{\underbrace{\alpha_t}_{d_k-1},
    \underbrace{\mu_t}_{d_k-1},\lambda_+,\lambda_-\}.
  \]
  \begin{itemize}
    \item Key-aligned dynamics have trace $r_t$ and determinant $\alpha_t\mu_t$.
    \item GDN also has orthogonal eigenvalues $\alpha_t$; $\alpha_t(1-\beta_t)$ is only its key-aligned eigenvalue.
    \item For $d_k>1$ and standard positive gates, GDN's spectral radius is $\alpha_t$.
  \end{itemize}
  \papersource{Paper, Appendix F, Eqs. (58)--(60), PDF p. 23. Last two bullets correct the spectral-radius statement on p. 5.}
\end{frame}

\begin{frame}{Backup: the reported floor also permits right-half-plane roots}
  For unit keys, $0<\alpha<1$, $\beta\le1-\alpha$, and $0<\eta<2$:
  \[
  \begin{aligned}
  r&=\alpha+\mu-\alpha\beta\eta\\
   &\ge\mu+\alpha-2\alpha(1-\alpha)\\
   &=\mu+2\left(\alpha-\tfrac14\right)^2-\tfrac18
   \ge\mu-\tfrac18.
  \end{aligned}
  \]
  \begin{block}{A sufficient condition derived for this reading}
  $\mu>1/8$ gives positive trace and determinant, hence positive real parts.
  The reported floor satisfies $e^{-2}\approx0.1353>1/8$.
  \end{block}
  This reconciles the $-2$ experimental floor with the spectral intent.
  It is a frozen-step result, not a proof of end-to-end training stability.
  \papersource{Reading derivation from Appendix F, Eq. (60), p. 23; constraints in Sec. 3.3, pp. 6--7; reported setting in Table 4, p. 9.}
\end{frame}

\begin{frame}{Backup: eliminating momentum reveals the second-order rule}
  For $\beta_{t-1}\ne0$, substitute
  $\Mmat_{t-1}=(\alpha_{t-1}\Smat_{t-2}-\Smat_{t-1})/\beta_{t-1}$:
  \[
  \begin{aligned}
  \Smat_t={}&
  \left[
    \left(\alpha_t+\frac{\beta_t\mu_t}{\beta_{t-1}}\right)\mathbf I
    -\alpha_t\beta_t\eta_t\kk_t\kk_t^\top
  \right]\Smat_{t-1}\\
  &-\frac{\beta_t\mu_t\alpha_{t-1}}{\beta_{t-1}}\Smat_{t-2}
  +\beta_t\eta_t\kk_t\vv_t^\top.
  \end{aligned}
  \]
  \begin{itemize}
    \item The additional $\Smat_{t-2}$ term explains the second-order interpretation.
    \item Table 1's summary omits $\alpha_t$ in the key-dependent transition term.
    \item The two-state recurrence avoids dividing by a previous step size.
  \end{itemize}
  \papersource{Reading derivation from Eqs. (4)--(5), p. 4; compare Table 1, p. 3, with the consistent block matrix in Appendix F, p. 23.}
\end{frame}

\end{document}