% Self-authored paper discussion; build with pdflatex, shell escape disabled.
% Main talk: frames 1--22, approximately 25 minutes. Remaining frames are backup.
\documentclass[aspectratio=169,11pt]{beamer}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{lmodern}
\usepackage{amsmath,amssymb,graphicx,booktabs}
\definecolor{AcademicBlue}{RGB}{26,57,86}
\definecolor{AcademicTeal}{RGB}{0,105,112}
\setbeamercolor{structure}{fg=AcademicBlue}
\setbeamercolor{frametitle}{fg=AcademicBlue,bg=white}
\setbeamercolor{title}{fg=AcademicBlue}
\setbeamercolor{alerted text}{fg=AcademicTeal}
\setbeamercolor{block title}{fg=white,bg=AcademicBlue}
\setbeamercolor{block body}{fg=black,bg=AcademicBlue!5}
\setbeamertemplate{navigation symbols}{}
\setbeamertemplate{itemize items}[circle]
\setbeamertemplate{footline}{\hfill\usebeamerfont{page number in head/foot}\insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}}
\setbeamersize{text margin left=8mm,text margin right=8mm}
\setbeamerfont{frametitle}{size=\Large,series=\bfseries}
\setbeamerfont{title}{size=\LARGE,series=\bfseries}
\setlength{\tabcolsep}{5pt}
\renewcommand{\arraystretch}{1.17}
\newcommand{\papersource}[1]{\par\smallskip{\scriptsize\color{gray}#1\par}}
\newcommand{\mrr}{M\textsuperscript{2}RNN}
\newcommand{\HH}{\mathbf H}
\newcommand{\ZZ}{\mathbf Z}
\newcommand{\WW}{\mathbf W}
\title[M2RNN]{\mrr: Non-Linear RNNs with Matrix-Valued States for Scalable Language Modeling}
\subtitle{Research paper discussion}
\author{Paper authors: Mayank Mishra, Shawn Tan, Ion Stoica,\\Joseph E. Gonzalez, Tri Dao}
\date{arXiv:2603.14360v2 \;|\; Revised 13 May 2026\\Reading: 9 October 2026}
\hypersetup{pdftitle={M2RNN: research paper discussion},pdfauthor={Paper reading notes},colorlinks=true,urlcolor=AcademicTeal}
\begin{document}

\begin{frame}
\titlepage
\vspace{-5mm}
\begin{center}\color{AcademicTeal}Large nonlinear memory becomes practical when used sparingly.\end{center}
\end{frame}

\begin{frame}{Three bottlenecks motivate one architecture}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.32\textwidth}
\begin{block}{State computation}
Nonlinear transitions can implement rich step-by-step updates.
\end{block}
Linear recurrences with restricted transitions have expressivity limits.
\end{column}
\begin{column}{0.32\textwidth}
\begin{block}{Memory capacity}
Outer-product writes expand a vector state into a matrix.
\end{block}
A small vector state can be a serious language-modeling bottleneck.
\end{column}
\begin{column}{0.32\textwidth}
\begin{block}{Hardware efficiency}
Matrix states provide useful GEMM tiles at small batch sizes.
\end{block}
Time steps remain sequential; sparse use controls the cost.
\end{column}
\end{columns}
\vspace{4mm}
\alert{The strongest practical result: add a few nonlinear layers to an existing hybrid.}
\papersource{Paper, Secs. 1--3, PDF pp. 1--10.}
\end{frame}

\begin{frame}{Write, transform, retain, and read a matrix state}
\[
\begin{aligned}
\ZZ_t &= \tanh(\HH_{t-1}\WW+k_tv_t^\top),\\
\HH_t &= f_t\HH_{t-1}+(1-f_t)\ZZ_t,\\
y_t &= \HH_t^\top q_t+w_r\odot v_t.
\end{aligned}
\]
\begin{itemize}
\item $\HH_t,\ZZ_t\in\mathbb R^{K\times V}$; $\WW\in\mathbb R^{V\times V}$; $\HH_0=0$.
\item $q_t,k_t\in\mathbb R^K$, $v_t,w_r\in\mathbb R^V$; $f_t\in[0,1]$ is scalar per head.
\item $\WW$ mixes value channels within each row; $q_t$ combines rows at readout.
\end{itemize}
\papersource{Paper, Eqs. (10)--(12), PDF p. 7.}
\end{frame}

\begin{frame}{Most of the block can be computed before recurrence}
\begin{columns}[c,onlytextwidth]
\begin{column}{0.52\textwidth}
\includegraphics[width=\linewidth,height=5.5cm,keepaspectratio]{figures/fig-02-block.pdf}
\end{column}
\begin{column}{0.45\textwidth}
\begin{itemize}
\item $q,k,v$: linear projections, causal Conv1D (width 4), then SiLU.
\item Forget and output gates depend on layer inputs.
\item After readout: multiply by $g_t$, apply RMSNorm, then project out.
\item Pre-norm residual blocks surround the mixer and MLP/MoE.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 2 (cropped) and Eqs. (14)--(23), PDF p. 8.}
\end{frame}

\begin{frame}{Shared queries and keys buy more recurrent capacity}
\begin{block}{Multi-value formulation}
$N_q=N_k=1$, while $N_v=N_f=N_w=N$. Each value head retains its own $K\times V$ state.
\end{block}
\[
K=64,\quad V=16
\quad\Rightarrow\quad
\underbrace{KV=1{,}024}_{\text{state elements/head}},\qquad
\underbrace{V^2=256}_{\text{transition parameters/head}}.
\]
\[
P_{\rm MH}\approx N(2K+3V)d_m,
\qquad P_{\rm MV}\approx3NVd_m.
\]
\begin{itemize}
\item Reuse $q/k$ projections to allocate a larger state at a fixed projection budget.
\item These are approximate \emph{projection} counts; recurrent matrices and other parameters are excluded.
\end{itemize}
\papersource{Paper, Algorithm 1 and Appendix A / Table 8, PDF pp. 11, 29.}
\end{frame}

\begin{frame}{The gate preserves a direct path through time}
\[
a_{t,n}=(\mathbf W_fx_t)_n,\qquad
f_{t,n}=\exp\!\left[-\alpha_n\operatorname{softplus}(a_{t,n}+\beta_n)\right].
\]
\begin{itemize}
\item $\alpha_n$ controls decay; $\beta_n$ shifts the response. Use $\alpha_n\ge0$ to keep $f\in[0,1]$.
\item $f_t=1$: retain the state. $f_t=0$: use the nonlinear candidate, which still depends on history.
\item Initialize $\WW=\mathbf I$ and clip recurrent-state gradients during BPTT.
\end{itemize}
\begin{block}{Why input-only gating matters}
Gate values can be precomputed across time. The nonlinear state transition still needs sequential evaluation.
\end{block}
\papersource{Paper, Sec. 3.1, Eqs. (13), (17), PDF pp. 7--9. Interpretation of gate endpoints follows the recurrence.}
\end{frame}

\begin{frame}{Nonlinear models generalize on the tested $S_3$ task}
\begin{columns}[c,onlytextwidth]
\begin{column}{0.62\textwidth}
\includegraphics[width=\linewidth,height=4.8cm,keepaspectratio]{figures/fig-03-s3.pdf}
\end{column}
\begin{column}{0.35\textwidth}
\begin{itemize}
\item Train at length 128; evaluate to 512.
\item \mrr\ and GRU: reported accuracy $\ge99.5\%$.
\item DeltaProduct loses accuracy as length grows.
\end{itemize}
\end{column}
\end{columns}
The recurrence can embed a vector RNN in one row. The experiment tests $S_3$, whereas the complexity motivation discusses $S_5$.
\papersource{Paper, Theorem 1 / Fig. 3, PDF p. 9; proof in Appendix B, p. 30.}
\end{frame}

\begin{frame}{Efficient tiles do not remove the sequential dependency}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.48\textwidth}
\begin{block}{Forward}
Parallelize over batch and heads. Keep $\HH$ and $\WW$ on chip; scan time sequentially.
\end{block}
GEMM shape: $(K,V)\times(V,V)$.
\medskip

Unlike batch-based vector tiles, this shape does not require padding a small batch to 16.
\end{column}
\begin{column}{0.48\textwidth}
\begin{block}{Backward}
Recompute the trajectory, cache states in HBM, then run the reverse sweep.
\end{block}
Temporary state storage: $O(BTNKV)$.
\medskip

The current Triton backward kernel is memory-bandwidth bound.
\end{column}
\end{columns}
\vspace{4mm}
Core recurrent work: $O(BTNKV^2)$; linear in length at fixed state dimensions.
\papersource{Paper, Sec. 3.4 and Secs. 4.1--4.2 / Algorithms 1--2, PDF pp. 10--12.}
\end{frame}

\begin{frame}{Two model scales share a 100B-token training protocol}
\begin{center}
\begin{tabular}{lll}
\toprule
& Dense & Mixture of Experts \\
\midrule
Parameters & 410M scale & 7B total, 1.1B active \\
Layers & 24 & 40 \\
Training GPUs & 32 H100s & 64 H100s \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Nemotron-CC-v2 high-quality subset; identical data and order.
\item 4,096-token training sequences; global batch of 1,024 sequences.
\item AdamW, peak learning rate $3\times10^{-4}$; BF16 with FP32 gradient accumulation.
\item Transformer++ uses RoPE; linear recurrent and hybrid models use NoPE.
\end{itemize}
\papersource{Paper, Sec. 5 / 5.2, PDF p. 15. Table captions round active parameters to 1B.}
\end{frame}

\begin{frame}{The three-way hybrid changes only a few mixer layers}
\begin{center}
\begin{tabular}{lll}
\toprule
Design & Attention layers & Recurrent layers \\
\midrule
Homogeneous & None & One recurrent family \\
Two-way hybrid & 1 in every 8 & GDN, Mamba-2, or \mrr \\
Three-way hybrid & 1 in every 8 & GDN/Mamba-2 plus \mrr \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item \textbf{+1}: replace the first recurrent layer.
\item \textbf{+3 at 410M / +5 at 7B}: replace each layer immediately before attention.
\item All variants are trained from scratch. Layer count and layer placement both change across these comparisons.
\end{itemize}
\papersource{Paper, Sec. 5.1, PDF p. 15. GDN = Gated DeltaNet.}
\end{frame}

\begin{frame}{Pure \mrr\ reaches competitive language modeling}
\begin{center}
\begin{tabular}{lrrrr}
\toprule
& \multicolumn{2}{c}{410M} & \multicolumn{2}{c}{7B MoE} \\
Model & Wiki PPL $\downarrow$ & Avg. acc. $\uparrow$ & Wiki PPL $\downarrow$ & Avg. acc. $\uparrow$ \\
\midrule
RNN & 33.74 & 48.90 & 17.65 & 60.58 \\
GRU & 25.80 & 53.72 & 14.80 & 64.44 \\
Mamba-2 & 22.93 & 54.30 & 13.73 & 65.22 \\
GDN & 22.70 & 53.99 & 13.89 & 64.28 \\
\mrr & 22.92 & 54.24 & 13.80 & 65.17 \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item \mrr\ closes much of the gap between vector RNNs and modern recurrent models.
\item It remains slightly behind the best recurrent baseline on these metrics.
\end{itemize}
\papersource{Paper, Tables 1--2, PDF p. 16. Average over ten 0-shot downstream accuracy columns.}
\end{frame}

\begin{frame}{One nonlinear layer is already useful at 7B}
\begin{center}
\begin{tabular}{lrrr}
\toprule
7B MoE model & Wiki PPL $\downarrow$ & LAMBADA PPL $\downarrow$ & Avg. acc. $\uparrow$ \\
\midrule
Transformer++ & 14.94 & 17.26 & 61.76 \\
Hybrid Mamba-2 & 13.10 & 11.05 & 65.03 \\
Hybrid GDN & 13.51 & 11.06 & 64.64 \\
Hybrid \mrr & 13.00 & 10.58 & 65.56 \\
Hybrid GDN +1 & 13.07 & 10.33 & \textbf{65.80} \\
Hybrid GDN +5 & \textbf{12.85} & \textbf{10.29} & 65.66 \\
\bottomrule
\end{tabular}
\end{center}
\begin{block}{Single-layer change relative to Hybrid GDN}
WikiText: $-0.44$ PPL. Downstream average: $+1.16$ percentage points.
\end{block}
\papersource{Paper, Table 2, PDF p. 16. +1 / +5 denote the number of substituted \mrr\ layers.}
\end{frame}

\begin{frame}{At 410M, gains depend on the metric}
\begin{center}
\begin{tabular}{lrrr}
\toprule
410M model & Wiki PPL $\downarrow$ & LAMBADA PPL $\downarrow$ & Avg. acc. $\uparrow$ \\
\midrule
Hybrid GDN & 21.89 & 29.02 & 54.04 \\
Hybrid \mrr & 21.53 & 37.31 & 54.15 \\
Hybrid GDN +1 & 21.39 & 33.78 & 53.75 \\
Hybrid GDN +3 & \textbf{21.26} & \textbf{28.93} & \textbf{55.23} \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item One layer improves WikiText by 0.50 PPL, but worsens LAMBADA PPL and the downstream average.
\item Three layers improve all three reported aggregate metrics in this comparison.
\end{itemize}
\papersource{Paper, Table 1, PDF p. 16. This qualifies the prose claim of gains across all benchmarks.}
\end{frame}

\begin{frame}{Attention hybrids outperform pure recurrent retrieval}
\begin{center}
\begin{tabular}{lrr}
\toprule
Model & 410M mean score $\uparrow$ & 7B mean score $\uparrow$ \\
\midrule
Pure GDN & 30.8 & 42.2 \\
Pure \mrr & 26.9 & 37.9 \\
Transformer++ & 34.6 & 45.5 \\
Hybrid GDN & 43.1 & 51.4 \\
Hybrid \mrr & \textbf{46.9} & \textbf{55.6} \\
Hybrid GDN +1 & 45.5 & 54.2 \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Means cover SQuAD, NQ, DROP, TriviaQA, FDA, and SWDE.
\item Pure \mrr\ still trails GDN; the paper reports a roughly $3\times$ smaller recurrent state for \mrr.
\end{itemize}
\papersource{Paper, Tables 3--4 / Sec. 5.4.2, PDF p. 20. Scores follow each task's reported evaluation.}
\end{frame}

\begin{frame}{Single-needle retrieval shows distinct failure modes}
\centering
\includegraphics[width=\linewidth,height=4.35cm,keepaspectratio]{figures/fig-06-single-with-legend.pdf}
\par\medskip
\raggedright
At 410M, the +3 GDN hybrid retains more S-NIAH-2 accuracy at 16K. On S-NIAH-1, replacing GDN layers can reduce accuracy.
\papersource{Paper, Fig. 6 top row, PDF p. 18; original legend retained. Dashed line: 4,096-token training length.}
\end{frame}

\begin{frame}{Multiple associations favor more nonlinear layers}
\centering
\includegraphics[width=\linewidth,height=4.3cm,keepaspectratio]{figures/fig-06-multi-with-legend.pdf}
\par\medskip
\raggedright
At 16K, +3 has the highest accuracy in these 410M MQ/MK/MV panels. Absolute accuracy remains limited. The Mamba backbone also needs multiple layers for more reliable gains.
\papersource{Paper, Fig. 6 bottom row and discussion of Figs. 7--9, PDF pp. 17--19. MQ: multiple queries; MK: distracting keys; MV: multiple values.}
\end{frame}

\begin{frame}{LongBench gains are large, but architecture-specific}
\begin{center}
\begin{tabular}{lrrrr}
\toprule
& \multicolumn{2}{c}{410M} & \multicolumn{2}{c}{7B MoE} \\
Model & Score $\uparrow$ & Gain & Score $\uparrow$ & Gain \\
\midrule
Hybrid GDN & 15.4 & --- & 23.1 & --- \\
Hybrid \mrr & 15.4 & 0.0 & 24.6 & +1.5 \\
Hybrid GDN +1 & 19.5 & +4.1 & 30.4 & +7.3 \\
Hybrid GDN +3 / +5 & 23.0 & +7.6 & 31.2 & +8.1 \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item Eight selected tasks: summarization, few-shot learning, and code completion.
\item 7B Hybrid Mamba-2: $18.8\to13.1$ with one layer; $22.7$ with five.
\item 7B GDN +1: MultiNews drops $16.7\to6.0$, despite the higher mean.
\end{itemize}
\papersource{Paper, Tables 5--6, PDF p. 21. Gains computed from displayed means; scores are not a common accuracy metric.}
\end{frame}

\begin{frame}{Capacity is a compelling explanation, with a confound}
\begin{center}
\begin{tabular}{lrrrr}
\toprule
Model & Params & State size & Wiki PPL $\downarrow$ & LAMBADA PPL $\downarrow$ \\
\midrule
RNN & 406M & 1,360 & 33.74 & 317.38 \\
GRU & 474M & 1,360 & 25.80 & 63.83 \\
\mrr & 410M & 86,016 & 22.92 & 33.63 \\
\bottomrule
\end{tabular}
\end{center}
\begin{itemize}
\item About $63.25\times$ as many state elements as the vector baselines, at a parameter count close to RNN.
\item GRU improves substantially at the same state size: gating matters too.
\item Matrix state, read/write structure, and gate design change together. Table 7 does not isolate their individual effects.
\end{itemize}
\papersource{Paper, Table 7 / Sec. 5.6, PDF pp. 21--22. State-size ratio is calculated from the table.}
\end{frame}

\begin{frame}{Sparse use keeps training throughput close to GDN}
\begin{columns}[c,onlytextwidth]
\begin{column}{0.62\textwidth}
\includegraphics[width=\linewidth,height=5.3cm,keepaspectratio]{figures/fig-10-throughput.pdf}
\end{column}
\begin{column}{0.35\textwidth}
\begin{itemize}
\item 7B MoE, 8 HGX H100 GPUs, NVLink.
\item Pure \mrr\ is substantially slower.
\item The paper reports a throughput loss within 6\% at 16K for Hybrid GDN +1.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Fig. 10 / Sec. 5.7, PDF p. 22. Training throughput in billions of tokens/day; no +5 or decoding-throughput curve.}
\end{frame}

\begin{frame}{Tensor parallelism trades topology dependence for sync}
\begin{columns}[T,onlytextwidth]
\begin{column}{0.48\textwidth}
\begin{block}{Topology-aware}
One query/key group per TP GPU.
\end{block}
\begin{itemize}
\item $N_q=N_k=N_{TP}$.
\item Local normalization and local recurrent heads.
\item No added communication beyond standard TP.
\item Projection sizes depend on the training topology.
\end{itemize}
\end{column}
\begin{column}{0.48\textwidth}
\begin{block}{Topology-independent}
Share one query/key group globally.
\end{block}
\begin{itemize}
\item $N_q=N_k=1$.
\item Shard normalization; synchronize its statistics.
\item Extra AllReduces: 1 forward, 3 backward.
\item Parameter count stays independent of TP size.
\end{itemize}
\end{column}
\end{columns}
\papersource{Paper, Sec. 4.3 / Figs. 4--5, PDF pp. 12--14; Appendix C, pp. 30--31.}
\end{frame}

\begin{frame}{Three qualifications change how to use the result}
\begin{enumerate}
\item \textbf{Complexity separation is conditional.} The $\mathsf{TC}^0$ versus $\mathsf{NC}^1$ separation uses a conjecture and specific computational assumptions. The direct experiment is $S_3$.
\medskip
\item \textbf{Capacity and architecture are coupled.} The ablation changes more than state size; single- and multi-layer hybrids also use different layer locations.
\medskip
\item \textbf{Implementation matters.} Algorithm 2 omits future-state gradient accumulation and has an incorrect state-gradient update. The chain-rule form is in the backup slides.
\end{enumerate}
\papersource{Paper, Secs. 2, 5.1, 5.6 and Algorithm 2, PDF pp. 3--5, 12, 15, 21--22. Complexity qualification: Merrill et al., ICML 2024, Sec. 3. Pseudocode issue does not establish a bug in the training code.}
\end{frame}

\begin{frame}{The strongest takeaway is a modular architecture result}
\begin{block}{A practical result at 7B}
Hybrid GDN + one \mrr\ layer: $-0.44$ WikiText PPL, $+7.3$ LongBench points, and reported training-throughput loss within 6\% at 16K.
\end{block}
\begin{itemize}
\item Matrix states make nonlinear recurrence competitive in language modeling.
\item A few expensive recurrent layers can complement attention and efficient linear recurrence.
\item The best layer count depends on backbone, task, and training cost.
\end{itemize}
\medskip
\href{https://arxiv.org/abs/2603.14360v2}{Mishra et al., \emph{M\textsuperscript{2}RNN}, arXiv:2603.14360v2}\par
\papersource{Key evidence: Tables 2 and 6, Fig. 10, PDF pp. 16, 21--22. Figures in this deck are reproduced or cropped from this version.}
\end{frame}

\appendix
\begin{frame}{Backup: embedding a vector RNN in one matrix row}
Set $f_t=0$, $w_r=0$, and $q_t=k_t=e_1$. Starting from $\HH_0=0$:
\[
\HH_{t,1,:}=\tanh(\HH_{t-1,1,:}\WW+v_t^\top),
\qquad \HH_{t,i,:}=0\ (i>1).
\]
With $h_t=\HH_{t,1,:}^\top$,
\[
h_t=\tanh(\WW^\top h_{t-1}+v_t),\qquad y_t=h_t.
\]
\begin{itemize}
\item This embeds the vector recurrence in the abstract matrix recurrence.
\item The proof allows the endpoint $f_t=0$; the finite-parameter positive gate typically approaches that endpoint only in a limit.
\item Output gating and RMSNorm are outside this core simulation argument.
\end{itemize}
\papersource{Paper, Appendix B / Eq. (25), PDF p. 30; endpoint qualification follows Eq. (13), p. 7.}
\end{frame}

\begin{frame}{Backup: the recurrent gradient needs both paths}
For $\bar y_t=\HH_t^\top q_t$, let $u_t=\partial\mathcal L/\partial\bar y_t$ and $\mathbf C_t$ be the gradient from future time steps.
\[
\begin{aligned}
\mathbf R_t&=\mathbf C_t+q_tu_t^\top,\\
\mathbf D_t&=((1-f_t)\mathbf R_t)\odot(1-\ZZ_t\odot\ZZ_t).
\end{aligned}
\]
\[\boxed{\mathbf C_{t-1}=f_t\mathbf R_t+\mathbf D_t\WW^\top.}\]
\[
\partial_{f_t}\mathcal L=\langle\mathbf R_t,\HH_{t-1}-\ZZ_t\rangle_F,
\qquad
\partial_{\WW}\mathcal L=\sum_t\HH_{t-1}^\top\mathbf D_t.
\]
The printed Algorithm 2 omits $\mathbf C_t$ and overwrites the direct gradient branch with an incorrect product. Recomputing the trajectory also requires updating the local state each step.
\papersource{Reading analysis derived from Eqs. (10)--(12), PDF p. 7; compare Algorithm 2, p. 12. The value residual and outer block contribute additional gradients.}
\end{frame}

\begin{frame}{Backup: projection sharing explains the state budget}
\begin{center}
\begin{tabular}{lll}
\toprule
Pattern & Approximate projection count & State at budget $P$ \\
\midrule
Multi-head & $N(2K+3V)d_m$ & $\dfrac{PKV}{(2K+3V)d_m}$ \\
Multi-query/key & $N(K+2V)d_m$ & $\dfrac{PKV}{(K+2V)d_m}$ \\
Multi-value & $3NVd_m$ & $\dfrac{PK}{3d_m}$ \\
\bottomrule
\end{tabular}
\end{center}
\medskip
\begin{itemize}
\item At $K=64,V=16$, these approximations give $S_{\rm MV}/S_{\rm MH}\approx3.67$.
\item Multi-value's listed projections sum to $(2K+3NV+N)d_m$ before approximation.
\item Recurrent matrices add $NV^2$ parameters, outside the table's projection count.
\end{itemize}
\papersource{Paper, Appendix A / Table 8, PDF p. 29. Ratios and unapproximated sum calculated from the listed terms.}
\end{frame}

\begin{frame}{Backup: RMSNormTP synchronizes per-token statistics}
For a feature shard $D_n$ on GPU $n$:
\[
s=\left[\frac1d\sum_{n=1}^{N_{TP}}\sum_{j\in D_n}x_j^2\right]^{-1/2},
\quad y_i=w_ix_is.
\]
\[
p_i=w_i\nabla_{y_i}\mathcal L,
\qquad
r=\sum_{n=1}^{N_{TP}}\sum_{j\in D_n}p_jx_j,
\qquad
\nabla_{x_i}\mathcal L=sp_i-\frac{rs^3x_i}{d}.
\]
\begin{itemize}
\item AllReduce the squared-norm sum in forward and the inner product in backward.
\item Each statistic is scalar \emph{per token}; a batch of sequences has $BT$ such statistics.
\item Two further backward AllReduces aggregate the shared query/key activation gradients.
\end{itemize}
\papersource{Paper, Appendix C / Eqs. (26)--(33), PDF pp. 30--31; Sec. 4.3.2, p. 14. Epsilon omitted as in the paper.}
\end{frame}

\begin{frame}{Backup: 7B benefits on multi-association retrieval}
\centering
\includegraphics[width=\linewidth,height=4.3cm,keepaspectratio]{figures/fig-08-multi-with-legend.pdf}
\par\medskip
\raggedright
Original figure labels the multi-layer curve ``+3''; Sec. 5.1 and the 7B tables specify +5. The figure is retained without relabeling.
\papersource{Paper, Fig. 8 bottom row, PDF p. 19; original legend retained. Dashed line: 4,096-token training length.}
\end{frame}

\begin{frame}{Backup: references and author-provided resources}
\begin{itemize}
\item Mayank Mishra, Shawn Tan, Ion Stoica, Joseph E. Gonzalez, Tri Dao.\par
\emph{\mrr: Non-Linear RNNs with Matrix-Valued States for Scalable Language Modeling}.\par
\href{https://arxiv.org/abs/2603.14360v2}{arXiv:2603.14360v2}, revised 13 May 2026.
\medskip
\item William Merrill, Jackson Petty, Ashish Sabharwal.\par
\href{https://proceedings.mlr.press/v235/merrill24a.html}{The Illusion of State in State-Space Models}. ICML 2024.\par
Used to clarify the complexity-theoretic assumptions.
\medskip
\item Paper-provided resources:\par
\href{https://github.com/open-lm-engine/lm-engine}{lm-engine training framework}\quad
\href{https://github.com/open-lm-engine/accelerated-model-architectures}{Model and kernel repository}\par
\href{https://huggingface.co/collections/open-lm-engine/m2rnn}{M\textsuperscript{2}RNN model collection}
\end{itemize}
\end{frame}
\end{document}