% Self-authored reading presentation, based on the skill's Beamer template. \documentclass[aspectratio=169,11pt]{beamer} \usepackage[T1]{fontenc} \usepackage[utf8]{inputenc} \usepackage{lmodern} \usepackage{amsmath,amssymb,graphicx,booktabs} \definecolor{AcademicBlue}{RGB}{26,57,86} \definecolor{AcademicTeal}{RGB}{0,105,112} \setbeamercolor{structure}{fg=AcademicBlue} \setbeamercolor{frametitle}{fg=AcademicBlue,bg=white} \setbeamercolor{title}{fg=AcademicBlue} \setbeamercolor{alerted text}{fg=AcademicTeal} \setbeamercolor{block title}{fg=white,bg=AcademicBlue} \setbeamercolor{block body}{fg=black,bg=AcademicBlue!5} \setbeamertemplate{navigation symbols}{} \setbeamertemplate{itemize items}[circle] \setbeamertemplate{footline}{% \hfill\usebeamerfont{page number in head/foot}% \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}} \setbeamersize{text margin left=8mm,text margin right=8mm} \setbeamerfont{frametitle}{size=\Large,series=\bfseries} \setbeamerfont{title}{size=\LARGE,series=\bfseries} \newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}} \newcommand{\M}{\mathbf M} \newcommand{\vk}{\mathbf k} \newcommand{\vq}{\mathbf q} \newcommand{\vv}{\mathbf v} \newcommand{\vx}{\mathbf x} \newcommand{\Iw}{\mathcal I_t^w} \newcommand{\Ir}{\mathcal I_t^r} \title[SDM]{Sparse Delta Memory} \subtitle{Scaling the State of Linear RNNs through Sparsity} \author{Paper by Lo\"ic Cabannes et al.\\[2mm] \small Independent paper-reading presentation} \date{\href{https://arxiv.org/abs/2607.07386v1}{arXiv:2607.07386v1}\\ 8 September 2026 \enspace$\cdot$\enspace 20--30 minute group discussion} \begin{document} \begin{frame} \titlepage \begin{center} \alert{A larger recurrent memory improves recall when only a few slots are accessed.} \end{center} \end{frame} \begin{frame}{Can recurrent memory grow without dense updates?} \begin{columns}[c] \begin{column}{0.49\textwidth} \begin{itemize} \item Full attention retains a growing KV cache. \item GDN keeps a compact state but touches it densely. \item SDM increases stored capacity while limiting the slots accessed per token. \end{itemize} \medskip \alert{Capacity $N$ and access budgets $W,R$ become separate controls.} \end{column} \begin{column}{0.48\textwidth} \includegraphics[width=\linewidth,height=0.66\textheight,keepaspectratio]{figures/fig01-state-flops.pdf} \end{column} \end{columns} \papersource{Paper, Secs. 1--2, pp. 1--3; original Fig. 1, p. 1. Plot is state vs FLOPs, not measured throughput.} \end{frame} \begin{frame}{SDM routes writes and reads into a large explicit table} \centering \includegraphics[width=0.91\linewidth,height=0.66\textheight,keepaspectratio]{figures/fig02-architecture.pdf} \par\smallskip \small Purple/dashed: sparse operations. Unselected write slots remain unchanged. \papersource{Original Fig. 2, PDF p. 3, cropped from the final paper. The printed update and this diagram differ; see slide 6.} \end{frame} \begin{frame}{Product keys select addresses without scoring every slot} \[ \vk'_t=[\vk'_{1,t},\vk'_{2,t}]\in\mathbb R^{2\sqrt N}, \qquad s_{a,b}=k'_{1,t}[a]+k'_{2,t}[b] \] \begin{enumerate} \item Project the input into two score vectors of length $\sqrt N$. \item Keep each half's top-$k$; form only $k^2$ candidate sums. \item Select top-$W$ write addresses and top-$R$ read addresses. \item Apply softmax over selected scores to obtain sparse weights. \end{enumerate} \begin{block}{Addressing cost} $O(\sqrt N\,d+W^2+R^2)$; no need to materialize all $N$ scores. \end{block} \papersource{Paper, Secs. 2 and 3.1, pp. 3--4. The text specifies an outer sum; Fig. 2 uses an inconsistent product symbol.} \end{frame} \begin{frame}{Selected slots decay and update; other slots persist} For each selected write slot $i\in\Iw$, v1 prints: \[ \widetilde{\M}_t[i]=\alpha_t\M_{t-1}[i], \qquad \M_t[i]=\widetilde{\M}_t[i] +\beta_t k_t^{(i)}\bigl(\vv_t-\widetilde{\M}_t[i]\bigr). \] \[ i\notin\Iw:\ \M_t[i]=\M_{t-1}[i], \qquad \mathbf y_t=\sum_{i\in\Ir}q_t^{(i)}\M_t[i]. \] \begin{itemize} \item $\alpha_t$: per-head forget gate; $\beta_t$: input gate. \item $k_t^{(i)},q_t^{(i)}$: sparse write/read weights; $\vv_t$: value vector. \item Read after writing, then RMS-Norm, output gating, and head mixing. \end{itemize} \papersource{Paper, Sec. 3.1, p. 4, Eqs. (3)--(5). This is Eq. (4) as printed; its residual differs from Fig. 2.} \end{frame} \begin{frame}{A v1 inconsistency matters for reproducing the update} \begin{block}{Fig. 2 reads one shared prediction before writing} \[ \widehat{\vv}_t=\widetilde{\M}_t^\top\vk_t,\qquad \M_t[i]=\widetilde{\M}_t[i] +\beta_t k_t^{(i)}(\vv_t-\widehat{\vv}_t). \] \end{block} \begin{itemize} \item Eq. (4) instead subtracts the individual slot $\widetilde{\M}_t[i]$. \item The shared residual matches GDN and the key interactions in Appendix~A. \item These updates differ for general multi-slot keys. \end{itemize} \medskip \alert{The printed Eq. (4) does not establish the claimed exact dense GDN limit.} \papersource{Paper, Eq. (2), p. 2; Fig. 2, p. 3; Eq. (4) and Connection to GDN, p. 4; App. A, pp. 14--15. Code not audited.} \end{frame} \begin{frame}{Head count controls a cubic state expansion} \[ N=\left(\frac{d}{4H}\right)^2,\qquad d_{\mathrm v}=\frac dH,\qquad M_{\mathrm{size}}=HN d_{\mathrm v}=\frac{d^3}{16H^2}. \] \begin{columns}[t] \begin{column}{0.48\textwidth} \begin{block}{1.4B configuration} $d=1920,\ H=2$\\ $N=57{,}600$ slots per head\\ $553$M state elements over 5 global layers\\ $\approx 900\times$ GDN \end{block} \end{column} \begin{column}{0.48\textwidth} \begin{block}{8B configuration} $d=3840,\ H=2$\\ $N=230{,}400$ slots per head\\ $7.963$B state elements over 9 global layers\\ $\approx 3600\times$ GDN \end{block} \end{column} \end{columns} \papersource{Paper, Sec. 3.3, Eq. (6), p. 5; Table 1, p. 6. Slot counts and ratios calculated from configurations. Elements are not bytes.} \end{frame} \begin{frame}{Matched FLOPs exclude the cost of storing more parameters} \begin{itemize} \item Match projection widths: $d_{\mathrm{qk}}^{\mathrm{total}}=d/2$ and $d_{\mathrm v}^{\mathrm{total}}=d$. \item Set $W=R=64$; the paper reports extra PKM selection below 1\% of layer FLOPs. \item Default SDM learns $\M_0$, but the reported parameter count excludes this memory. \end{itemize} \medskip \centering \begin{tabular}{lrrr} \toprule Scale & Reported params & Learned state & Sum (derived)\\ \midrule 1.4B & 1.48B & 0.553B & 2.033B\\ 8B & 8.14B & 7.963B & 16.103B\\ \bottomrule \end{tabular} \par\medskip \alert{This is not a comparison at equal total learnable storage.} \papersource{Paper, Sec. 3.2, pp. 4--5; Table 1 and Sec. 5.1, p. 6. Sums use rounded reported counts.} \end{frame} \begin{frame}{Training parallelizes within chunks and recurs across chunks} \begin{columns}[t] \begin{column}{0.48\textwidth} \begin{block}{Batched work within chunks} \begin{itemize} \item Accumulate decay per accessed slot. \item Build sparse write--write and read--write interactions. \item Solve a triangular system for delta corrections. \end{itemize} \end{block} \end{column} \begin{column}{0.48\textwidth} \begin{block}{Sequential work across chunks} \begin{itemize} \item Gather the incoming memory. \item Correct, then scatter updates. \item Combine past-state reads with within-chunk contributions. \end{itemize} \end{block} \end{column} \end{columns} \medskip Backward undoes sparse writes instead of saving a full table at every chunk. \papersource{Paper, App. A.1--A.3, pp. 14--15. This describes the appendix's shared-residual mechanism; the v1 formula caveat remains.} \end{frame} \begin{frame}{The comparisons use a shared local/global hybrid} \begin{block}{Architecture} SWA (window 128) and global layers interleaved at roughly 3:1.\\ Replace the global layer with FullAttn, GDN, Mamba2, or SDM. \end{block} \begin{itemize} \item Pretrain at 8192 tokens; AdamW and a warmup--stable--decay schedule. \item Scale from 257M to 8.14B counted parameters at 160 tokens per non-embedding parameter. \item 1.4B: 168.4B pretraining tokens, then 4B tokens at 128k. \item 8B: 1.141T pretraining tokens, then 16B tokens at 128k. \end{itemize} \papersource{Paper, Sec. 4, pp. 5--6; Table 1, p. 6. FullAttn in the results is also a SWA/global hybrid.} \end{frame} \begin{frame}{SDM achieves lower training loss than GDN across the ladder} \begin{columns}[c] \begin{column}{0.66\textwidth} \includegraphics[width=\linewidth,height=0.70\textheight,keepaspectratio]{figures/fig03-scaling.pdf} \end{column} \begin{column}{0.31\textwidth} \begin{itemize} \item Fit levels 1--8; predict level 13. \item SDM fit: $R^2=0.999$. \item At 8B, SDM also beats FullAttn in training loss. \end{itemize} \medskip \small This fit does not establish benchmark significance. \end{column} \end{columns} \papersource{Paper, Sec. 5.1, p. 6; original Fig. 3, p. 7. Loss averages the last 10 log entries. Memory parameters excluded from matching.} \end{frame} \begin{frame}{Short-context averages improve, with substantial task variation} \centering \begin{tabular}{llrrr} \toprule Scale & Metric & FullAttn & GDN & SDM\\ \midrule 1.4B & Text NLL $\downarrow$ & 2.660 & 2.683 & \textbf{2.654}\\ & Average score (\%) $\uparrow$ & 37.98 & 37.93 & \textbf{38.51}\\ \midrule 8B & Text NLL $\downarrow$ & 2.285 & 2.298 & \textbf{2.253}\\ & Average score (\%) $\uparrow$ & 56.65 & 55.70 & \textbf{56.84}\\ & BoolQ (\%) $\uparrow$ & 71.83 & \textbf{74.98} & 67.13\\ \bottomrule \end{tabular} \par\medskip \begin{itemize} \item 8B: +1.14 percentage points vs GDN; +0.19 vs FullAttn. \item BoolQ drops 7.85 points vs GDN. No benchmark error bars reported. \end{itemize} \papersource{Paper, Table 2, p. 8; post-trained models, 15-task average. 8B NLL difference recalculates to -0.045, not the printed -0.040.} \end{frame} \begin{frame}{RULER improves over GDN; FullAttn still leads overall} \centering \begin{tabular}{lrrrr} \toprule Scale & FullAttn & GDN & SDM & SDM minus GDN\\ \midrule 1.4B & \textbf{32.5} & 20.0 & 31.2 & +11.2 pp\\ 8B & \textbf{61.2} & 34.2 & 50.2 & +16.0 pp\\ \bottomrule \end{tabular} \par\bigskip \begin{block}{Protocol} Exact-match accuracy (\%), averaged over 13 tasks and lengths 4k--131k, after 128k fine-tuning. \end{block} \begin{itemize} \item 8B SDM remains 11.0 points below FullAttn. \item The body also reports 76.2 for FullAttn under an inconsistent task discussion; this slide uses Table 2. \end{itemize} \papersource{Paper, Sec. 5.3, pp. 6--7; Table 2, p. 8. pp = percentage points.} \end{frame} \begin{frame}{Hard multi-key retrieval remains a clear weakness} \centering \begin{tabular}{lrrr} \toprule 8B RULER task, accuracy (\%) & FullAttn & GDN & SDM\\ \midrule single\_3 & 70.1 & 32.6 & \textbf{74.9}\\ multiquery & \textbf{73.0} & 31.2 & 68.6\\ variable tracking & 67.2 & 46.2 & \textbf{72.3}\\ multikey\_2 & \textbf{58.0} & 1.0 & 4.7\\ multikey\_3 & \textbf{29.8} & 0.1 & 1.2\\ \bottomrule \end{tabular} \par\bigskip SDM improves or matches GDN on all 13 tasks at 8B.\\[1mm] At 1.4B, however, frequent-word extraction falls from 35.6 to 12.3. \papersource{Paper, Table 2, p. 8. Selected task rows; averages span lengths. Full length curves: Figs. 9--10, p. 17.} \end{frame} \begin{frame}{More context improves code modeling into very long documents} \begin{columns}[c] \begin{column}{0.53\textwidth} \includegraphics[width=\linewidth,height=0.70\textheight,keepaspectratio]{figures/fig04-code-ppl.pdf} \end{column} \begin{column}{0.44\textwidth} \begin{itemize} \item 1.4B models; 1M-token code documents. \item Solid: 128k fine-tuned. Dashed: pretrained only. \item SDM approaches PPL 2.0; dense RNNs stay around 2.2--2.3. \item Raw PPL rises after 256k. Authors attribute this to local token difficulty. \end{itemize} \end{column} \end{columns} \papersource{Paper, Sec. 5.4, p. 7; original Fig. 4, p. 9. Values are approximate. The claimed isolated global-layer gain is not plotted here.} \end{frame} \begin{frame}{Capacity helps recall even without a learned initial memory} \centering \begin{tabular}{llrrr} \toprule 1.4B model & $\M_0$ & Code NLL $\downarrow$ & Avg. score (\%) & RULER (\%)\\ \midrule GDN & zero & 0.849 & 37.9 & 20.0\\ GDN & learned & 0.850 & 38.0 & 20.6\\ SDM & zero & 0.845 & 37.3 & 28.0\\ SDM & learned & \textbf{0.822} & \textbf{38.5} & \textbf{31.2}\\ \bottomrule \end{tabular} \par\bigskip \begin{itemize} \item Zero-initialized SDM gains 8.0 RULER points over zero-initialized GDN. \item Learning SDM's initial table adds another 3.2 points and lowers code NLL. \item Learned $\M_0$ supplies pretrained information; context updates modify $\M_t$ during inference. \end{itemize} \papersource{Paper, Sec. 6, pp. 7--9; Table 3 and Fig. 5, p. 9. Accuracy columns: higher is better.} \end{frame} \begin{frame}{Shrinking the memory gradually degrades NLL and recall} \centering \begin{tabular}{lrrrr} \toprule 0.8B model & State/layer & Code NLL & Avg. score (\%) & RULER (\%)\\ \midrule GDN, zero $\M_0$ & 0.2 MB & 0.963 & 33.8 & 16.0\\ SDM, learned $\M_0$ & 27 MB & 0.947 & 33.7 & 20.2\\ SDM, learned $\M_0$ & 108 MB & 0.937 & 33.6 & 20.7\\ SDM, learned $\M_0$ & 432 MB & \textbf{0.914} & \textbf{34.6} & \textbf{21.5}\\ \bottomrule \end{tabular} \par\bigskip \begin{itemize} \item Larger state improves code NLL ($\downarrow$) and RULER ($\uparrow$) monotonically here. \item Short-task averages are not strictly monotonic. \item Varying the key projection width changes $N$; this is not a pure byte-capacity intervention. \end{itemize} \papersource{Paper, Sec. 6, pp. 7--8; Table 3 (bottom), p. 9. MB labels retained from paper; $W,R,d_{\mathrm v}$ and non-SDM layers fixed.} \end{frame} \begin{frame}{Sparse arithmetic still pays for random memory access} \begin{block}{The hardware bottleneck} GDN's small state fits in SRAM. SDM's large table resides in HBM. \end{block} \begin{itemize} \item Current SDM kernel MFU is about one order of magnitude below the optimized GDN kernel. \item 8B end-to-end training: reported as $1.49\times$ slower than GDN. \item 1.4B decode: about 10\% slower than GDN, about $6\times$ faster than FullAttn in the reported test. \item 8B SDM state: as much memory as 203,400 tokens of the compared FullAttn KV cache. \end{itemize} \papersource{Paper, Sec. 6, pp. 8--9, and Sec. 7, p. 10. Full batch/length timing protocol is not reported; these are not universal speed ratios.} \end{frame} \begin{frame}{Read weights spread more broadly than write weights} \begin{columns}[c] \begin{column}{0.49\textwidth} \centering \includegraphics[width=\linewidth,height=0.46\textheight,keepaspectratio]{figures/fig06a-write-mass.pdf} \par\smallskip Writes: top 32 of 64 carry $\sim85\%$. \end{column} \begin{column}{0.49\textwidth} \centering \includegraphics[width=\linewidth,height=0.46\textheight,keepaspectratio]{figures/fig06b-read-mass.pdf} \par\smallskip Reads: top 64 of 128 carry $\sim77\%$. \end{column} \end{columns} \[ C(m)=\sum_{i=1}^{m}p_{(i)},\qquad p_{(1)}\geq p_{(2)}\geq\cdots \] \small Curves: mean cumulative mass; shading: token-level p10--p90, not training-run confidence intervals. \papersource{Paper, Sec. 6, p. 9; original Fig. 6(a,b), p. 10; App. D.1, p. 18. Approximate coverage depends on configuration.} \end{frame} \begin{frame}{Full attention and SDM show complementary behavior} \centering \begin{tabular}{lrrr} \toprule 1.4B pretrained & SWA:FA & SWA:(FA/SDM) & SWA:SDM\\ \midrule Text NLL $\downarrow$ & 2.658 & \textbf{2.645} & 2.654\\ Code NLL $\downarrow$ & 0.798 & \textbf{0.797} & 0.821\\ Average score (\%) $\uparrow$ & 38.1 & \textbf{38.6} & \textbf{38.6}\\ RULER (\%) $\uparrow$ & \textbf{80.1} & 64.3 & 56.9\\ \bottomrule \end{tabular} \par\bigskip \begin{itemize} \item Alternate global layers: FA--SDM--FA--SDM--FA. \item RULER here covers only 6 tasks at 4k--8k; do not compare it directly with the 13-task main table. \item The mixed model retains a growing global KV cache. \end{itemize} \papersource{Paper, App. E, p. 19; Table 5, p. 20. Fig. 11 fits levels 1--8; it does not report a trained 8B mixed model.} \end{frame} \begin{frame}{The evidence supports a useful tradeoff, with clear boundaries} \begin{itemize} \item \textbf{Capacity:} larger fixed state helps, but difficult multi-key tasks remain weak. \item \textbf{Comparison:} parameter matching excludes learned memory; matched FLOPs do not imply equal wall time. \item \textbf{Reproducibility:} the update equation and several numerical statements need clarification in v1. \item \textbf{Scope:} text/code experiments reach 8B and million-position code modeling; long-lived agents and video memory remain future directions. \end{itemize} \papersource{Paper, Secs. 5--7, pp. 6--10; App. A--F, pp. 14--20. Interpretation based on reported experiments; no independent reproduction.} \end{frame} \begin{frame}{Three points to retain} \begin{enumerate} \item \textbf{Scale capacity through sparse access.}\\ A large recurrent table can be useful without updating every element at each token. \medskip \item \textbf{Separate dynamic state from learned initialization.}\\ Both contribute, and they answer different memory questions. \medskip \item \textbf{Evaluate storage and real speed alongside FLOPs.}\\ The next bottleneck is maintaining and accessing a large per-sequence state. \end{enumerate} \papersource{Reading synthesis. Main evidence: Tables 1--3, pp. 6, 8--9; efficiency and limitations, pp. 8--10.} \end{frame} \begin{frame}{Paper identity and materials used} \textbf{Sparse Delta Memory: Scaling the State of Linear RNNs through Sparsity} \par\medskip Lo\"ic Cabannes, Pierre-Emmanuel Mazar\'e, Gergely Szilvasy, Matthijs Douze, Maria Lomeli, Ilze Amanda Auzina, Justin Carpentier, Gabriel Synnaeve, Herv\'e J\'egou. \par\medskip \href{https://arxiv.org/abs/2607.07386v1}{arXiv:2607.07386v1}, submitted 8 July 2026. \begin{itemize} \item Read same-version author TeX, final PDF, and original figures. \item Covered main text and Appendices A--F. \item All experimental figures are original; selected table rows preserve paper values. \item This presentation is a reading analysis; authors' code was not run or audited. \end{itemize} \papersource{Retrieved 8 September 2026. PDF cover date is 9 July 2026. Local companion: reading-notes.zh.md; figure provenance: figures/manifest.json.} \end{frame} \appendix \begin{frame}{Backup: two slots expose the update discrepancy} \[ \alpha=0.8,\quad \beta=0.5,\quad \M_{t-1}=[2.5,\ 5]^\top,\quad \widetilde{\M}_t=[2,\ 4]^\top,\quad \vk=[0.5,\ 0.5]^\top,\quad v=10. \] \begin{block}{Eq. (4): individual slot residual} $[2,\ 4]+0.25\,[10-2,\ 10-4]=[4,\ 5.5]$. \end{block} \begin{block}{Fig. 2: shared prediction residual} $\widehat v=0.5(2)+0.5(4)=3$.\\ $[2,\ 4]+0.25\,[10-3,\ 10-3]=[3.75,\ 5.75]$. \end{block} \papersource{Teaching counterexample constructed for this reading; not experimental data. Compare Eq. (4), p. 4, with Fig. 2, p. 3.} \end{frame} \begin{frame}{Backup: sparse interactions drive the triangular solve} \[ \mathbf A[i,j]= \sum_{\substack{n,m:\ \mathcal I_i^w[n]=\mathcal I_j^w[m]}} k_i^{(n)}k_j^{(m)} e^{\lambda_{i,n}-\lambda_{j,m}},\qquad j