\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{%
  \hfill\usebeamerfont{page number in head/foot}%
  \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\keypoint}[1]{{\color{AcademicTeal}\bfseries #1}}
\newcommand{\fig}[3]{\includegraphics[width=#1,height=#2,keepaspectratio]{#3}}

\begin{document}

\begin{frame}[plain]
  \vspace{0.9cm}
  {\LARGE\bfseries\color{AcademicBlue} RAM-Net: Linear-Time Sequence Modeling\\[2mm]
  with Sparsely Addressable State\par}
  \vspace{0.7cm}
  {\large Paper discussion \quad|\quad 3 November 2026\par}
  \vspace{0.5cm}
  Paper authors: Kaicheng Xiao, Haotian Li, Liran Dong, Guoliang Xing\par
  \vspace{0.35cm}
  \href{https://arxiv.org/abs/2602.11958v2}{arXiv:2602.11958v2}
  \quad $\bullet$ \quad Accepted at NeurIPS 2026
  (\href{https://neurips.cc/Downloads/2026}{official list})\par
  \vspace{0.18cm}
  \href{https://github.com/MuonCat/RAM-Net}{Official public implementation}\par
  \vfill
  {\color{AcademicTeal}\large Takeaway: sparse slot addressing improves fine-grained recall
  with low per-token state access.}
\end{frame}

\begin{frame}{The question: can a fixed state support precise recall?}
  \begin{block}{The tension}
    Full attention keeps separable token entries; recurrent linear attention
    compresses them into a shared fixed-size state.
  \end{block}
  \begin{itemize}
    \item Shared writes superpose unrelated values.
    \item Dense additive reads bring interference into the output.
    \item Enlarging a densely accessed state raises per-step access cost.
  \end{itemize}
  \medskip
  \keypoint{RAM-Net's proposal:} keep a bounded state, partition it into
  slots, and sparsify both writes and reads.
  \papersource{Paper, Secs. 1--2, PDF pp. 1--3.}
\end{frame}

\begin{frame}{RAM-Net changes the unit of storage and access}
  \centering
  \fig{0.98\linewidth}{0.62\textheight}{figures/new_intro_cmp.png}
  \vspace{2mm}
  \begin{itemize}
    \item Full attention: per-token KV entries; score all cached keys.
    \item Linear attention: shared state; dense update and read.
    \item RAM-Net: fixed slot array; only selected slots are touched.
  \end{itemize}
  \papersource{Original paper Fig. 1, PDF p. 3.}
\end{frame}

\begin{frame}{One layer separates address, update, and readout}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.62\textwidth}
      \centering
      \fig{\linewidth}{0.69\textheight}{figures/fig2-block.png}
    \end{column}
    \begin{column}{0.36\textwidth}
      \begin{enumerate}
        \item Project $x_t$ to $k_t,q_t,v_t,\gamma_t$.
        \item Decode sparse write/read addresses.
        \item Update addressed slots with GSU.
        \item Read, normalize, gate, and project.
      \end{enumerate}
    \end{column}
  \end{columns}
  \papersource{Cropped from original paper Fig. 2, PDF p. 4; Sec. 3.}
\end{frame}

\begin{frame}{Product Softmax addresses $d_p^U$ slots with $Ud_p$ logits}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.47\textwidth}
      \centering
      \fig{\linewidth}{0.58\textheight}{figures/fig3-digits.png}
    \end{column}
    \begin{column}{0.51\textwidth}
      \[
        a=\sum_{u=0}^{U-1}a^{(u)}d_p^u,\qquad
        p^{(u)}=\operatorname{softmax}(k_t^{(u)}/\tau).
      \]
      \[
        \rho_U(k_t)[a]
          =\prod_{u=0}^{U-1}p^{(u)}[a^{(u)}].
      \]
      \begin{itemize}
        \item $M=|\mathcal A|=d_p^U$ slots per head.
        \item $d_k=Ud_p$ key/query dimensions.
        \item Example: $U=5,d_p=4 \Rightarrow M=1024,\ d_k=20$.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 3.2, Eq. (4), PDF pp. 4--5; image cropped from Fig. 3.}
\end{frame}

\begin{frame}{Exact Top-$K$ search never builds the full address vector}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.47\textwidth}
      \centering
      \fig{\linewidth}{0.67\textheight}{figures/fig3-tree.png}
    \end{column}
    \begin{column}{0.51\textwidth}
      \begin{itemize}
        \item Retain the best $K$ prefixes at each digit.
        \item Extend each by $d_p$ choices, then keep the best $K$.
        \item All prefixes share the same remaining digit distributions:
        a pruned prefix cannot return to the final Top-$K$.
      \end{itemize}
      \[
        \text{decode cost} =
        O(UKd_p+Ud_p\log d_p).
      \]
      Computation uses log probabilities; retained weights are
      \keypoint{not renormalized}.
    \end{column}
  \end{columns}
  \papersource{Paper, Secs. 3.2 and 4, Eq. (5), PDF pp. 5--6; image cropped from Fig. 3.}
\end{frame}

\begin{frame}{CAPE makes address overlap depend on relative position}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.51\textwidth}
      \centering
      \fig{\linewidth}{0.60\textheight}{figures/fig2-address.png}
    \end{column}
    \begin{column}{0.47\textwidth}
      \[
        \pi_t(a)=(a-t)\bmod M.
      \]
      \[
        \begin{aligned}
        w_t[\pi_t(a)]&=\mathcal D_{K,U}(k_t)[a],\\
        r_t[\pi_t(a)]&=\mathcal D_{K,U}(q_t)[a].
        \end{aligned}
      \]
      \begin{itemize}
        \item A write at $s$ and read at $t$ overlap according
        to content \emph{and} distance $t-s$.
        \item Some heads use CAPE; others use content-only NoPE.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Paper, Sec. 3.3, Eq. (6), PDF p. 5; cropped from Fig. 2.}
\end{frame}

\begin{frame}{GSU updates only addressed slots}
  For each slot $a$, keep content $s_t[a]\in\mathbb R^{d_v}$ and
  write mass $m_t[a]$:
  \[
  \begin{aligned}
  \lambda_t[a]&=\sigma\!\left(\operatorname{logit}(1-w_t[a])-\gamma_t\right),\\
  m_t[a]&=\lambda_t[a]m_{t-1}[a]+w_t[a],\\
  \eta_t[a]&=\frac{w_t[a]}{m_t[a]+\epsilon},\\
  s_t[a]&=s_{t-1}[a]+\eta_t[a](v_t-s_{t-1}[a]).
  \end{aligned}
  \]
  $w_t[a]$: write weight; $\gamma_t$: input-dependent forgetting strength;
  $\eta_t[a]$: adaptive update fraction.
  \papersource{Paper, Sec. 3.4, Eqs. (7)--(8), PDF p. 6.}
\end{frame}

\begin{frame}{The sparse update has three useful consequences}
  \begin{enumerate}
    \item \keypoint{Exact inactivity:} if $w_t[a]=0$, then
    $\lambda_t[a]=1,\ \eta_t[a]=0$; the slot is unchanged.
    \item \keypoint{Bounded content:} $0\leq\eta_t[a]<1$ makes each
    update a convex combination of old content and $v_t$.
    \item \keypoint{Adaptive retention:} larger retained mass slows
    new writes; larger $\gamma_t$ reduces retention.
  \end{enumerate}
  \medskip
  \begin{block}{Readout and interpretation}
    $o_t=\sum_a r_t[a]s_t[a]$, followed by per-head RMSNorm and an output gate.
    Each slot update is also a gradient step on
    $\frac12\|s-v_t\|_2^2$ with step size $\eta_t[a]$.
  \end{block}
  \papersource{Paper, Sec. 3.4, PDF p. 6.}
\end{frame}

\begin{frame}{Access cost is decoupled from total state capacity}
  \begin{center}
    \begin{tabular}{lll}
      \toprule
      Quantity & Scaling per head & Meaning\\
      \midrule
      Slot count & $M=d_p^U$ & fixed during a sequence\\
      State storage & $O(M(d_v+1))$ & content plus write mass\\
      Address search & $O(UKd_p+Ud_p\log d_p)$ & no $M$-wide scan\\
      State touch & $O(Kd_v)$ & sparse read/write\\
      \bottomrule
    \end{tabular}
  \end{center}
  \begin{itemize}
    \item Training/prefill: decode in parallel, group events by slot,
    preserve time order within each slot.
    \item Autoregressive decoding: update/read the selected slots.
  \end{itemize}
  \papersource{Paper, Sec. 4, PDF p. 6. Scaling describes the recurrent core at fixed architecture hyperparameters.}
\end{frame}

\begin{frame}{Evaluation spans language modeling and three recall tests}
  \begin{center}
    \begin{tabular}{p{0.28\linewidth}p{0.62\linewidth}}
      \toprule
      Training & FineWeb-Edu; Llama 2 tokenizer; 4,096-token context\\
      Scales & 340M parameters / 10B tokens; 1.3B / 100B\\
      Baselines & Transformer++, Mamba2, GLA, GSA, Gated DeltaNet\\
      Language & WikiText-103 ppl $\downarrow$;
      zero-shot task average accuracy $\uparrow$\\
      Recall & MQAR; RULER S-NIAH at 1K--16K; nine real-document
      recall columns $\uparrow$\\
      Systems & 340M decode throughput on one RTX 5090 $\uparrow$\\
      \bottomrule
    \end{tabular}
  \end{center}
  \papersource{Paper, Sec. 5 and Appendix A, PDF pp. 7--9, 14.}
\end{frame}

\begin{frame}{At 340M, quality rises with $K$ while access remains small}
  \begin{center}
  \begin{tabular}{lrrr}
    \toprule
    Model & Wiki ppl $\downarrow$ & Commonsense avg. $\uparrow$ &
      Active state $\downarrow$\\
    \midrule
    Transformer++ & 26.81 & 49.4 & 100.7M\\
    Mamba2 & 26.23 & 50.6 & 12.9M\\
    Gated DeltaNet & 26.85 & 50.1 & 8.5M\\
    RAM-Net Top-8 & 26.89 & 50.3 & 0.4M\\
    \textbf{RAM-Net Top-32} & \textbf{25.87} & \textbf{50.9} & 1.6M\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Top-8 touches about $32\times$ fewer state elements than Mamba2;
    this is an access count, \emph{not} a throughput ratio.
    \item The total RAM-Net state is 26.2M elements at this scale.
  \end{itemize}
  \papersource{Paper, Table 1 and footnote, PDF p. 7. Transformer++ active state measured at length 2,048.}
\end{frame}

\begin{frame}{At 1.3B, RAM-Net matches the best average accuracy}
  \begin{center}
  \begin{tabular}{lrrr}
    \toprule
    Model & Wiki ppl $\downarrow$ & Commonsense avg. $\uparrow$ &
      Active state $\downarrow$\\
    \midrule
    Transformer++ & 17.08 & 56.0 & 201.3M\\
    Gated DeltaNet & 17.09 & 56.4 & 17.0M\\
    Mamba2 & 15.96 & \textbf{57.8} & 26.0M\\
    RAM-Net Top-8 & 16.49 & 57.4 & 0.8M\\
    \textbf{RAM-Net Top-16} & \textbf{15.90} & \textbf{57.8} & 1.6M\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Top-16 has the lowest reported perplexity by a small margin
    and ties Mamba2 on the average accuracy.
    \item Total RAM-Net state is 52.4M elements at this scale.
  \end{itemize}
  \papersource{Paper, Table 1 and footnote, PDF p. 7.}
\end{frame}

\begin{frame}{MQAR favors sparse addressing at a given state size}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.54\textwidth}
      \centering
      \fig{\linewidth}{0.70\textheight}{figures/zoology_result.png}
    \end{column}
    \begin{column}{0.44\textwidth}
      \begin{itemize}
        \item MQAR: synthetic key--value recall; models trained
        from scratch.
        \item RAM-Net Top-8/16 lies near the upper envelope
        by \emph{total} state size.
        \item Pale/dark RAM-Net markers denote
        \emph{active}/\emph{total} state.
      \end{itemize}
      \medskip
      \keypoint{Interpretation:} partitioned slots help limit
      interference between unrelated writes.
    \end{column}
  \end{columns}
  \papersource{Original paper Fig. 4, PDF p. 7; Sec. 5.}
\end{frame}

\begin{frame}{S-NIAH averages favor RAM-Net at both model scales}
  RULER single-needle accuracy averages 3 task types $\times$ 5 lengths
  (1K--16K). All models trained at 4K context.
  \medskip
  \begin{center}
  \begin{tabular}{lrrr}
    \toprule
    Scale & Transformer++ & Best other recurrent & RAM-Net\\
    \midrule
    340M & 48.6 & Gated DeltaNet 51.3 & \textbf{Top-32 72.9}\\
    1.3B & 56.4 & Mamba2 62.4 & \textbf{Top-16 70.1}\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item S-NIAH-1 has repeated filler; RAM-Net stays near 100\%
    through 16K.
    \item The more diverse S-NIAH-2/3 settings are substantially harder.
  \end{itemize}
  \papersource{Paper, Table 2 and Sec. 5, PDF p. 8. Values are accuracy percentages.}
\end{frame}

\begin{frame}{The difficult 16K tail exposes an addressing trade-off}
  \begin{center}
  \begin{tabular}{lrrrr}
    \toprule
     & \multicolumn{2}{c}{S-NIAH-2} & \multicolumn{2}{c}{S-NIAH-3}\\
    \cmidrule(lr){2-3}\cmidrule(lr){4-5}
    Model & 8K & 16K & 8K & 16K\\
    \midrule
    340M RAM-Net Top-32 & 56.3 & 4.0 & 9.7 & 1.0\\
    1.3B RAM-Net Top-16 & 49.0 & 18.7 & 18.3 & 6.3\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{block}{Why this matters}
    Sparse access reduces collision, but a read misses relevant content
    if its query does not overlap the original write slots.
  \end{block}
  The paper links the easy S-NIAH-1 case to repeated filler sharing
  addresses; diverse filler makes interference and routing harder.
  \papersource{Paper, Table 2 and Sec. 5, PDF p. 8; limitation discussed on PDF p. 10. Values are percentages.}
\end{frame}

\begin{frame}{Real-document recall still trails full attention}
  \begin{center}
  \begin{tabular}{lrrr}
    \toprule
    Scale & Transformer++ & Best other recurrent & RAM-Net\\
    \midrule
    340M & \textbf{35.1} & Mamba2 26.9 & Top-32 32.0\\
    1.3B & \textbf{49.8} & Gated DeltaNet 39.5 & Top-8 40.6\\
    \bottomrule
  \end{tabular}
  \end{center}
  \begin{itemize}
    \item Average over FDA, SWDE, NQ at 512/1K, plus full-document
    SQuADv2, TriviaQA, and DROP.
    \item RAM-Net leads the listed recurrent models at both scales,
    while per-token KV storage retains an advantage on document detail.
  \end{itemize}
  \papersource{Paper, Table 3, PDF p. 8; Sec. 5, PDF p. 9. Values are average accuracy percentages.}
\end{frame}

\begin{frame}{Sparse access translates into competitive decoding speed}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.48\textwidth}
      \centering
      \fig{\linewidth}{0.70\textheight}{figures/decode_tps.png}
    \end{column}
    \begin{column}{0.50\textwidth}
      \begin{itemize}
        \item 340M models; one NVIDIA RTX 5090.
        \item Contexts from 1K to 32K; output in tokens/s.
        \item RAM-Net Top-8 stays near GLA and above Mamba2,
        DeltaNet, and Gated DeltaNet across the shown range.
      \end{itemize}
      \medskip
      GLA is fast with a small \emph{total} state;
      RAM-Net accesses a small part of a larger state.
    \end{column}
  \end{columns}
  \papersource{Original paper Fig. 5, PDF p. 9; Sec. 5. Transformer++ is not shown in this plot.}
\end{frame}

\begin{frame}{Ablations identify more than a slot-count effect}
  \begin{center}
  \begin{tabular}{lrrr}
    \toprule
    340M variant & Wiki ppl $\downarrow$ &
      S-NIAH-2 @4K $\uparrow$ & S-NIAH-3 @4K $\uparrow$\\
    \midrule
    Top-8 default & 26.89 & 83.7 & 18.3\\
    Top-32 & \textbf{25.87} & \textbf{94.0} & \textbf{44.7}\\
    No output RMSNorm & 28.87 & 16.3 & 12.3\\
    NoPE only & 28.04 & 43.0 & 8.7\\
    Renormalize Top-$K$ & 26.83 & 63.7 & 12.0\\
    Slot balancing loss & 27.10 & 11.3 & 5.7\\
    \bottomrule
  \end{tabular}
  \end{center}
  \keypoint{Reading:} readout scaling, position-aware heads, and
  uneven slot usage all contribute to retrieval.
  \papersource{Paper, Table 5, PDF pp. 15--16. Values are percentages except perplexity.}
\end{frame}

\begin{frame}{Access traces show complementary head behavior}
  \centering
  \fig{0.98\linewidth}{0.59\textheight}{figures/addr_trace.png}
  \begin{itemize}
    \item CAPE head: diagonal drift from position-dependent shifts.
    \item NoPE head: vertical bands of reused, content-driven slots.
  \end{itemize}
  \papersource{Original paper Fig. 7, PDF p. 16; interpretation in Appendix C, PDF p. 17.}
\end{frame}

\begin{frame}{Take-home: sparse access buys recall, with routing limits}
  \begin{enumerate}
    \item A fixed state can be partitioned into many addressable slots.
    Product Softmax and exact Top-$K$ select a few efficiently.
    \item CAPE and GSU make position-aware access and sparse forgetting
    compatible with the slot array.
    \item Reported retrieval gains are strong on MQAR and S-NIAH
    averages; language modeling remains competitive.
  \end{enumerate}
  \medskip
  \begin{block}{Main boundary in the reported evidence}
    Queries can miss relevant write slots. Difficult 16K retrieval
    remains weak in absolute terms; real-document recall trails full attention.
    Experiments stop at 1.3B parameters.
  \end{block}
  \papersource{Paper, Secs. 5, 7--8, PDF pp. 7--10.}
\end{frame}

\begin{frame}{Sources}
  \begin{itemize}
    \item Kaicheng Xiao, Haotian Li, Liran Dong, Guoliang Xing.
    \emph{RAM-Net: Linear-Time Sequence Modeling with Sparsely Addressable State}.
    \href{https://arxiv.org/abs/2602.11958v2}{arXiv:2602.11958v2}, 2026.
    \item Acceptance status:
    \href{https://arxiv.org/abs/2602.11958v2}{authors' arXiv v2 comment};
    \href{https://neurips.cc/Downloads/2026}{NeurIPS 2026 official list}.
    \item \href{https://github.com/MuonCat/RAM-Net}{Official public code repository}.
  \end{itemize}
  \vspace{0.6cm}
  {\color{AcademicTeal}Original paper figures appear on slides 3--7, 14,
  18, 20, and 23; cropped panels are identified on their slides.}
\end{frame}

\appendix
\begin{frame}{Backup: product structure also helps optimization}
  \begin{columns}[T,onlytextwidth]
    \begin{column}{0.52\textwidth}
      \centering
      \fig{\linewidth}{0.68\textheight}{figures/zoology_ablation.png}
    \end{column}
    \begin{column}{0.46\textwidth}
      \begin{itemize}
        \item MQAR accuracy generally rises with Product Softmax
        order $U$, especially at larger slot counts.
        \item With $U=1$, a single $M$-way Softmax gives selected
        entries a sparse direct learning signal.
        \item With $U>1$, selected slots share digit distributions.
      \end{itemize}
    \end{column}
  \end{columns}
  \papersource{Original paper Fig. 6 and Appendix B, PDF pp. 14--15.}
\end{frame}

\begin{frame}{Backup: training and initialization details}
  \begin{itemize}
    \item Both scales use FineWeb-Edu, Llama 2 tokenizer,
    AdamW, 4K context, cosine decay, and 0.5M-token global batches.
    \item 340M: 10B tokens, peak LR $10^{-3}$, 500M warmup tokens.
    \item 1.3B: 100B tokens, peak LR $4\times10^{-4}$,
    1B warmup tokens.
    \item Per-digit, per-head learnable temperature correction begins
    from base $\tau=2.2$.
    \item Biases initialize output gate near open, accelerate early
    forgetting, and break address symmetry.
  \end{itemize}
  \papersource{Paper, Appendix A, Table 4, PDF p. 14.}
\end{frame}

\end{document}