% Generated for the arXiv-paper-reader workflow; compile with pdflatex. \documentclass[aspectratio=169,11pt]{beamer} \usepackage[T1]{fontenc} \usepackage[utf8]{inputenc} \usepackage{amsmath,amssymb,graphicx,booktabs} \definecolor{AcademicBlue}{RGB}{26,57,86} \definecolor{AcademicTeal}{RGB}{0,105,112} \setbeamercolor{structure}{fg=AcademicBlue} \setbeamercolor{frametitle}{fg=AcademicBlue,bg=white} \setbeamercolor{title}{fg=AcademicBlue} \setbeamercolor{alerted text}{fg=AcademicTeal} \setbeamercolor{block title}{fg=white,bg=AcademicBlue} \setbeamercolor{block body}{fg=black,bg=AcademicBlue!5} \setbeamertemplate{navigation symbols}{} \setbeamertemplate{itemize items}[circle] \setbeamertemplate{footline}{% \hfill\usebeamerfont{page number in head/foot}% \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}} \setbeamersize{text margin left=8mm,text margin right=8mm} \setbeamerfont{frametitle}{size=\Large,series=\bfseries} \setbeamerfont{title}{size=\LARGE,series=\bfseries} \newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}} \title[FG2-GDN]{FG\texorpdfstring{$^2$}{2}-GDN: Enhancing Long-Context Gated Delta Networks\\with Doubly Fine-Grained Control} \subtitle{Channel-wise writing rates for recurrent attention memory} \author{Paper by Pingwei Sun et al. (Meituan)} \date{arXiv:2604.19021v3 \quad | \quad November 6, 2026} \begin{document} \begin{frame} \titlepage \begin{center} \small Published at \href{https://aclanthology.org/2026.emnlp-main.2242/}{EMNLP 2026} \quad Official code: not found (checked 2026-11-06) \end{center} \end{frame} \begin{frame}{The paper gives the write gate the same granularity as forgetting} \begin{block}{Research question} KDA learns a separate forgetting rate for every channel, but still shares one delta-rule writing rate across all channels. Can writing become channel-wise without losing KDA's efficient chunkwise algorithm? \end{block} \begin{itemize} \item FG$^2$-GDN predicts a channel-wise $\beta_t$ and uses it to scale keys and values symmetrically. \item FG$^2$-GDN+ adds separate key and value rates, $\beta_t^k$ and $\beta_t^v$. \item At 1.3B, FG$^2$-GDN leads the language-modeling average; FG$^2$-GDN+ leads aggregate RULER and LongBench. \end{itemize} \papersource{Paper, Abstract and Secs. 1, 4, PDF pp. 1, 5--9.} \end{frame} \begin{frame}{KDA refines forgetting; FG$^2$-GDN refines writing} \centering \begin{tabular}{lcc} \toprule Method & Forgetting rate $\alpha_t$ & Writing rate $\beta_t$ \\ \midrule GDN & Scalar & Scalar \\ KDA & Channel-wise vector & Scalar \\ FG$^2$-GDN & Channel-wise vector & Channel-wise vector \\ FG$^2$-GDN+ & Channel-wise vector & Separate channel-wise $\beta_t^k,\beta_t^v$ \\ \bottomrule \end{tabular} \vspace{4mm} \begin{itemize} \item $\alpha_t$ controls how much old state is retained. \item $\beta_t$ controls the strength of the current delta update. \item The contribution is an orthogonal design axis to KDA's channel-wise decay. \end{itemize} \papersource{Paper, Secs. 2.2--3.1, PDF pp. 2--3.} \end{frame} \begin{frame}{The delta rule corrects a fast associative memory} A linear-attention state stores key--value associations and is read by the query: \[ o_t=S_t^\top q_t,\qquad S_t\in\mathbb{R}^{d_k\times d_v}. \] \vspace{-1mm} The delta update takes one gradient step on the reconstruction loss \[ \mathcal{L}_t(S)=\tfrac12\lVert S^\top k_t-v_t\rVert^2, \qquad S_t=(I-\beta_t k_tk_t^\top)S_{t-1}+\beta_t k_tv_t^\top. \] \begin{itemize} \item $S_{t-1}^\top k_t$ is the memory's current prediction for key $k_t$. \item The rank-1 correction moves that prediction toward $v_t$. \item Gating adds data-dependent forgetting so the state can discard old associations. \end{itemize} \papersource{Paper, Sec. 2.1, PDF p. 2, Eqs. (5)--(7).} \end{frame} \begin{frame}{FG$^2$-GDN makes the writing rate channel-wise} For each head, let $\alpha_t,\beta_t\in(0,1)^{d_k}$ and define \[ \tilde{k}_t=\sqrt{\beta_t}\odot k_t, \qquad \tilde{v}_t=\sqrt{\beta_t}\odot v_t. \] \[ S_t=(I-\tilde{k}_t\tilde{k}_t^\top)\,\mathrm{Diag}(\alpha_t)S_{t-1} +\tilde{k}_t\tilde{v}_t^\top, \qquad o_t=S_t^\top q_t. \] \begin{itemize} \item $\alpha_t$ still chooses what to forget in each key dimension. \item $\beta_t$ now chooses how strongly each dimension participates in the update. \item In the model, a low-rank projection predicts $\beta_t$ from the token representation. \end{itemize} \papersource{Paper, Secs. 3.2 and 3.5, PDF pp. 3--4, Eqs. (10)--(11).} \end{frame} \begin{frame}{Square-root scaling preserves KDA's tied rank-1 correction} \begin{columns}[T] \column{0.48\textwidth} \textbf{Naive vector replacement} \[ \mathrm{Diag}(\beta_t)k_tk_t^\top =(\beta_t\odot k_t)k_t^\top \] Left and right factors differ, so the symmetric tied form used by KDA's specialized WY-based algorithm is lost. \column{0.48\textwidth} \textbf{FG$^2$-GDN} \[ \tilde{k}_t\tilde{k}_t^\top =\mathrm{Diag}(\sqrt{\beta_t})k_tk_t^\top \mathrm{Diag}(\sqrt{\beta_t}) \] The correction remains symmetric rank-1, so KDA's chunkwise structure is retained. \end{columns} \vspace{2mm} \begin{block}{Important interpretation} Channel-wise does not mean an unconstrained learning rate for every matrix entry: pairwise channel effects remain coupled by the rank-1 factorization. \end{block} \papersource{Paper, Sec. 3.2--3.3, PDF p. 3, Eqs. (10)--(12).} \end{frame} \begin{frame}{FG$^2$-GDN+ separates erasing from writing values} \[ \hat{k}_t=\sqrt{\beta_t^k}\odot k_t, \qquad \tilde{v}_t=\sqrt{\beta_t^v}\odot v_t, \] \[ S_t=(I-\hat{k}_t\hat{k}_t^\top)\,\mathrm{Diag}(\alpha_t)S_{t-1} +\hat{k}_t\tilde{v}_t^\top. \] \begin{itemize} \item $\beta_t^k$ controls erasure and the key-side factor of the write. \item $\beta_t^v$ independently scales the value-side write. \item The extra gate projection adds a small parameter/FLOP cost while preserving the update structure. \end{itemize} \papersource{Paper, Sec. 3.4--3.5, PDF p. 4, Eq. (13).} \end{frame} \begin{frame}{The channel-wise gate reuses KDA's chunkwise training path} \begin{itemize} \item Expand the transition as $\mathrm{Diag}(\alpha_t)+a_tb_t^\top$, with $a_t=-\tilde{k}_t$ and $b_t=\tilde{k}_t\odot\alpha_t$. \item The tied diagonal-plus-rank-1 form allows the same WY representation and UT transform used by KDA. \item For sequence length $L$, chunk size $C$, head dimension $d$: \[ O(Ld^2+LCd+LC^2). \] \item Added elementwise scaling is $O(Ld)$, estimated at about 0.8\% of dominant state-update arithmetic for $d_k=d_v=128$. \end{itemize} \begin{block}{Measured overhead} FG$^2$-GDN prefill throughput is within 5\% of KDA in all tested settings; the separate value gate adds a little more cost. \end{block} \papersource{Paper, Secs. 3.6 and 4.5, PDF pp. 4, 7--8.} \end{frame} \begin{frame}{A 3:1 linear-attention/MLA hybrid tests the memory cell} \centering \includegraphics[width=0.96\linewidth,height=0.78\textheight,keepaspectratio]{figures/fig-01-architecture.pdf} \papersource{Paper, Fig. 1, PDF p. 5. Three linear-attention layers are followed by one MLA layer.} \end{frame} \begin{frame}{Experiments hold the backbone and training recipe fixed} \begin{columns}[T] \column{0.49\textwidth} \textbf{Two model scales} \begin{itemize} \item 340M: 20 layers, hidden size 1024, 15B tokens. \item 1.3B: 24 layers, hidden size 2048, 100B tokens. \item 16 heads; linear-attention head dimension 128. \end{itemize} \column{0.49\textwidth} \textbf{Shared protocol} \begin{itemize} \item 3:1 linear attention to MLA; same SlimPajama data and schedule. \item 8K training length; BF16, AdamW, cosine schedule. \item Peak LR $3\times10^{-4}$; each configuration trained once. \end{itemize} \end{columns} \vspace{2mm} \begin{itemize} \item Language modeling: LAMBADA perplexity plus seven zero-shot accuracy scores. \item Long context: RULER at 4K/8K/16K and LongBench, both at 1.3B. \end{itemize} \papersource{Paper, Sec. 4 and Appendix A, PDF pp. 5, 11.} \end{frame} \begin{frame}{Channel-wise writing improves the language-modeling average} \centering \small \begin{tabular}{llrr} \toprule Scale & Method & Accuracy avg. $\uparrow$ & LAMBADA PPL $\downarrow$ \\ \midrule 340M & GDN & 43.51 & 39.86 \\ & KDA & 43.14 & 41.30 \\ & FG$^2$-GDN & \textbf{44.88} & 38.27 \\ & FG$^2$-GDN+ & 44.84 & \textbf{37.93} \\ \midrule 1.3B & GDN & 52.79 & 14.01 \\ & KDA & 53.19 & 13.80 \\ & FG$^2$-GDN & \textbf{53.95} & \textbf{13.09} \\ & FG$^2$-GDN+ & 53.40 & 13.91 \\ \bottomrule \end{tabular} \vspace{2mm} \begin{itemize} \item The mean is over seven task accuracies; perplexity is reported separately. \item At 1.3B, FG$^2$-GDN is +0.76 percentage points over KDA on mean accuracy. \end{itemize} \papersource{Paper, Table 1, PDF p. 6.} \end{frame} \begin{frame}{FG$^2$-GDN+ leads RULER at all tested lengths} \centering \begin{tabular}{lrrr} \toprule 1.3B model & 4K & 8K & 16K \\ \midrule GDN & 61.6 & 51.0 & 42.4 \\ KDA & 61.5 & 55.0 & 44.9 \\ FG$^2$-GDN & 64.8 & 57.3 & 44.6 \\ FG$^2$-GDN+ & \textbf{67.2} & \textbf{61.4} & \textbf{48.9} \\ \bottomrule \end{tabular} \vspace{4mm} \begin{itemize} \item Scores are average accuracy (\%) over 13 RULER task categories. \item Models train at 8K; 16K evaluates extrapolation beyond training length. \item Some individual tasks remain non-monotonic across methods. \end{itemize} \papersource{Paper, Sec. 4.2 and Table 2, PDF pp. 5--6.} \end{frame} \begin{frame}{Separating key and value control lifts LongBench overall} \centering \begin{tabular}{lr} \toprule 1.3B model & LongBench average (\%) $\uparrow$ \\ \midrule GDN & 15.1 \\ KDA & 16.4 \\ FG$^2$-GDN & 16.0 \\ FG$^2$-GDN+ & \textbf{18.3} \\ \bottomrule \end{tabular} \vspace{4mm} \begin{block}{Read the ablation, not only the final score} FG$^2$-GDN improves language modeling but is slightly below KDA on LongBench. The full FG$^2$-GDN+ variant gives the strongest aggregate long-context score; code-completion subtasks still favor KDA. \end{block} \papersource{Paper, Sec. 4.2 and Table 3, PDF pp. 5, 7.} \end{frame} \begin{frame}{Ablations isolate both sources of gain} \centering \small \begin{tabular}{lccc} \toprule Variant & LM avg. $\uparrow$ & PPL $\downarrow$ & LongBench $\uparrow$ \\ \midrule GDN & 52.79 & 14.01 & 15.1 \\ Scalar $\alpha$, vector $\beta$ (FGBGDN) & 53.25 & 13.96 & 16.8 \\ KDA (vector $\alpha$) & 53.19 & 13.80 & 16.4 \\ FG$^2$-GDN (vector $\alpha,\beta$) & \textbf{53.95} & \textbf{13.09} & 16.0 \\ Vector $\alpha$, separate scalar $\beta^{k/v}$ & 53.9 & 13.56 & 16.7 \\ FG$^2$-GDN+ (both refinements) & 53.40 & 13.91 & \textbf{18.3} \\ \bottomrule \end{tabular} \vspace{3mm} \begin{itemize} \item Vector $\beta$ helps over GDN and further improves language modeling over KDA. \item Key/value separation boosts LongBench, especially when combined with vector $\beta$. \end{itemize} \papersource{Paper, Sec. 4.4 and Table 4, PDF p. 7.} \end{frame} \begin{frame}{The learned writing rate varies across channels} \centering \includegraphics[width=0.96\linewidth,height=0.79\textheight,keepaspectratio]{figures/fig-02-beta-stats.png} \papersource{Paper, Sec. 4.3 and Fig. 2, PDF p. 7. 1,408 held-out tokens; 18 linear-attention layers.} \end{frame} \begin{frame}{Long prefill keeps the linear-attention speed advantage} \centering \includegraphics[width=0.95\linewidth,height=0.74\textheight,keepaspectratio]{figures/fig-03-prefill-speed.pdf} \papersource{Paper, Sec. 4.5 and Fig. 3, PDF p. 8. H800-80G, BF16; 32,768 total tokens per setting.} \end{frame} \begin{frame}{Evidence is promising at 340M--1.3B; deployment remains open} \begin{columns}[T] \column{0.57\textwidth} \textbf{What the evidence supports} \begin{itemize} \item Channel-wise $\beta$ adds a useful writing-control axis to KDA's channel-wise decay. \item FG$^2$-GDN+ has the best aggregate RULER and LongBench scores reported. \item The structured update preserves chunkwise training and competitive prefill speed. \end{itemize} \column{0.40\textwidth} \textbf{Authors' stated limits} \begin{itemize} \item Dense hybrids at 340M and 1.3B; one pretraining run per setting. \item No 7B+, MoE, seed variance, per-method tuning, or KDA warm start. \item Prefill is measured; serving and speculative-decoding integration remain open. \end{itemize} \end{columns} \papersource{Paper, Conclusion and Limitations, PDF p. 9.} \end{frame} \begin{frame}{Takeaways} \begin{enumerate} \item KDA makes forgetting channel-wise; FG$^2$-GDN also makes writing channel-wise. \item Symmetric square-root scaling retains a tied rank-1 transition and KDA's chunkwise algorithm. \item Decoupling $\beta^k$ from $\beta^v$ gives the strongest aggregate long-context results in this study. \item The gains are demonstrated in 340M/1.3B hybrid pretraining, with measured prefill efficiency on H800. \end{enumerate} \begin{block}{One-sentence summary} Fine-grained write control improves what the recurrent memory can learn per token while preserving its efficient update structure. \end{block} \papersource{Paper, Sec. 6, PDF p. 9.} \end{frame} \begin{frame}{Paper and publication} \begin{itemize} \item Pingwei Sun et al. 2026. ``FG$^2$-GDN: Enhancing Long-Context Gated Delta Networks with Doubly Fine-Grained Control.'' \item Read version: \href{https://arxiv.org/abs/2604.19021v3}{arXiv:2604.19021v3}. \item Published in \href{https://aclanthology.org/2026.emnlp-main.2242/}{Proceedings of EMNLP 2026}, pp. 51013--51023. \item Official/public implementation matching the paper: not found as of 2026-11-06. \end{itemize} \vspace{3mm} \textbf{Related methods cited by the paper:} DeltaNet; Gated DeltaNet; Kimi Delta Attention; RULER; LongBench. \end{frame} \end{document}