% Based on the arxiv-paper-reader Beamer template. % Build: python3 build_slides.py slides.en.tex \documentclass[aspectratio=169,11pt]{beamer} \usepackage[T1]{fontenc} \usepackage[utf8]{inputenc} \usepackage{lmodern} \usepackage{amsmath,amssymb,graphicx,booktabs} \definecolor{AcademicBlue}{RGB}{26,57,86} \definecolor{AcademicTeal}{RGB}{0,105,112} \setbeamercolor{structure}{fg=AcademicBlue} \setbeamercolor{frametitle}{fg=AcademicBlue,bg=white} \setbeamercolor{title}{fg=AcademicBlue} \setbeamercolor{alerted text}{fg=AcademicTeal} \setbeamercolor{block title}{fg=white,bg=AcademicBlue} \setbeamercolor{block body}{fg=black,bg=AcademicBlue!5} \setbeamertemplate{navigation symbols}{} \setbeamertemplate{itemize items}[circle] \setbeamertemplate{footline}{% \hfill\usebeamerfont{page number in head/foot}% \insertframenumber\,/\,\inserttotalframenumber\hspace{4mm}\vspace{3mm}} \setbeamersize{text margin left=8mm,text margin right=8mm} \setbeamerfont{frametitle}{size=\Large,series=\bfseries} \setbeamerfont{title}{size=\LARGE,series=\bfseries} \setbeamerfont{subtitle}{size=\large} \setlength{\tabcolsep}{7pt} \renewcommand{\arraystretch}{1.18} \newcommand{\papersource}[1]{\par\medskip{\scriptsize\color{gray}#1\par}} \newcommand{\takeaway}[1]{\begin{block}{Interpretation}#1\end{block}} \newcommand{\RR}{\mathbb R} \newcommand{\bH}{\mathbf H} \newcommand{\bh}{\mathbf h} \newcommand{\bB}{\mathbf B} \newcommand{\bC}{\mathbf C} \newcommand{\bX}{\mathbf X} \newcommand{\bY}{\mathbf Y} \newcommand{\bR}{\mathbf R} \newcommand{\bQ}{\mathbf Q} \title[Mamba-3]{Mamba-3} \subtitle{Improved Sequence Modeling\\using State Space Principles} \author{Reading presentation\\[2mm] {\small Paper: Aakash Lahoti, Kevin Y. Li, Berlin Chen, Caitlin Wang,}\\ {\small Aviv Bick, J. Zico Kolter, Tri Dao, and Albert Gu}} \date{\small arXiv:2603.15569v1 \quad March 16, 2026\\ Prepared October 20, 2026 \quad 25-minute group discussion} \begin{document} \begin{frame} \titlepage \vspace{-4mm} \centering {\color{AcademicTeal}Richer state updates at a competitive decoding cost.} \par\smallskip {\scriptsize\href{https://arxiv.org/abs/2603.15569v1}{arxiv.org/abs/2603.15569v1}} \end{frame} \begin{frame}{Make each fixed-state update do more useful work} \begin{itemize} \item Fixed recurrent state avoids a growing KV cache. \item Simple positive decay limits state-transition behavior. \item Low-FLOP decoding can still leave GPU compute idle. \end{itemize} \medskip \centering \begin{tabular}{lll} \toprule Design change & What changes & Main objective\\ \midrule Exponential-trapezoidal & Input integration & Better local mixing\\ Complex transitions & State rotation & State tracking\\ MIMO & Read/write rank & Quality per decode cost\\ \bottomrule \end{tabular} \takeaway{Optimize the content of an update as well as its memory footprint.} \papersource{Paper, Secs. 1 and 3, PDF pp. 1--2, 4--11.} \end{frame} \begin{frame}{Mamba-3 changes the recurrence and its surrounding block} \begin{columns}[c] \begin{column}{0.65\textwidth} \includegraphics[width=\linewidth,height=0.67\textheight,keepaspectratio]{figures/fig-02-architecture.pdf} \end{column} \begin{column}{0.32\textwidth} \begin{itemize} \item BC normalization and learned biases. \item Data-dependent rotary embeddings. \item Optional MIMO projections. \item External short convolution removed. \end{itemize} \end{column} \end{columns} \papersource{Original Fig. 2; Sec. 3.4, PDF pp. 11--12. Blocks alternate with pre-norm SwiGLU.} \end{frame} \begin{frame}{Mamba-2 uses positive scalar decay and rank-one writes} \[ \bh_t=\alpha_t\bh_{t-1}+\Delta_t\bB_t x_t, \qquad y_t=\bC_t^\top\bh_t \] \[ \alpha_t=e^{\Delta_t A_t}\in(0,1),\qquad A_t<0,\quad\Delta_t>0 \] \begin{itemize} \item $N$: real state coordinates. $P$: channels per head. \item Stack $P$ channels: $\bH_t\in\RR^{N\times P}$ and $\Delta_t\bB_t\mathbf x_t^\top$ is a rank-one write. \item A larger $\Delta_t$ increases forgetting and current-input weight. \end{itemize} \takeaway{The update can scale memory, but it cannot directly rotate it.} \papersource{Paper, Sec. 2.2, Eq. (1), PDF p. 3; Sec. 3.3, p. 9.} \end{frame} \begin{frame}{Discretization separates decay from input integration} Freeze the scalar $A(t)$ at the right endpoint $A_t$: \[ \bh_t\approx e^{\Delta_t A_t}\bh_{t-1} +\int_{\tau_{t-1}}^{\tau_t} e^{(\tau_t-\tau)A_t}\bB(\tau)x(\tau)\,d\tau. \] \medskip \begin{columns}[t] \begin{column}{0.47\textwidth} \begin{block}{Right endpoint: exponential-Euler} \[ \text{input term}=\Delta_t\bB_t x_t \] Recovers the Mamba-1/2 implementation. \end{block} \end{column} \begin{column}{0.49\textwidth} \begin{block}{Two endpoints: exponential-trapezoidal} \[ \text{input term}=\beta_t\bB_{t-1}x_{t-1} +\gamma_t\bB_t x_t \] Learns how to mix adjacent state inputs. \end{block} \end{column} \end{columns} \papersource{Paper, Sec. 3.1.1, Eqs. (4)--(6), PDF pp. 5--6; App. A, p. 22.} \end{frame} \begin{frame}{The new recurrence contains a two-tap state-input filter} \[ \boxed{\bh_t=\alpha_t\bh_{t-1} +\beta_t\bB_{t-1}x_{t-1}+\gamma_t\bB_t x_t} \] \[ \alpha_t=e^{\Delta_t A_t},\quad \beta_t=(1-\lambda_t)\Delta_t\alpha_t,\quad \gamma_t=\lambda_t\Delta_t \] \centering \includegraphics[width=0.87\linewidth,height=0.29\textheight,keepaspectratio]{figures/fig-01-trapezoid.pdf} \begin{itemize} \item The filter acts on $\bB_t x_t$ inside the recurrence. \item SSD mask = decay matrix $\times$ two-band matrix. \end{itemize} \papersource{Paper, Fig. 1, PDF p. 4; Secs. 3.1.2--3.1.3, Eq. (7), pp. 6--7.} \end{frame} \begin{frame}{Higher quadrature order requires a condition on the gate} \[ \lambda_t=\tfrac12+O(\Delta_t) \quad\Longrightarrow\quad \text{input-integral error}=O(\Delta_t^3) \] \begin{itemize} \item Requires the smoothness and stability assumptions in Appendix A. \item Freezing a varying $A(t)$ already introduces $O(\Delta_t^2)$ local error. \item The default $\lambda_t=\sigma(u_t)$ does not enforce the condition. \end{itemize} \medskip \centering \begin{tabular}{lc} \toprule Gate, 440M component ablation & Perplexity $\downarrow$\\ \midrule Learned $\sigma(u_t)$ & \textbf{15.72}\\ Fixed $1/2$ & 15.76\\ Fixed $1$ (Euler) & 15.81\\ \bottomrule \end{tabular} \papersource{Paper, Remark 3, PDF p. 6; App. A.2--A.3, pp. 23--24, Table 8.} \end{frame} \begin{frame}{Complex transitions add input-dependent rotations} \[ e^{\Delta_t(A_t+i\theta_t)} =e^{\Delta_t A_t} \bigl(\cos(\Delta_t\theta_t)+i\sin(\Delta_t\theta_t)\bigr) \] \[ \bh_t=\alpha_t\bR_t\bh_{t-1}+\Delta_t\bB_t x_t,\qquad R(\phi)= \begin{bmatrix}\cos\phi&-\sin\phi\\ \sin\phi&\cos\phi\end{bmatrix} \] \begin{itemize} \item $N/2$ complex coordinates use $N$ real coordinates. \item Amplitude decay and phase evolution become separate controls. \item Teaching example: rotate by $\pi x_t$ to toggle a parity state. \end{itemize} \takeaway{The restricted positive transition is the obstacle. Negative real transitions can also solve parity.} \papersource{Paper, Prop. 2, Eqs. (8)--(9), PDF pp. 7--8; Table 5b, p. 14. Last point: algebraic interpretation.} \end{frame} \begin{frame}{A rotating coordinate system keeps the core update scalar} \[ \bQ_t=\prod_{s=0}^{t}\bR_s^\top,\quad \overline{\bB}_t=\bQ_t\bB_t,\quad \overline{\bC}_t=\bQ_t\bC_t,\quad \widetilde{\bh}_t=\bQ_t\bh_t \] \[ \boxed{\widetilde{\bh}_t =\alpha_t\widetilde{\bh}_{t-1} +\beta_t\overline{\bB}_{t-1}x_{t-1} +\gamma_t\overline{\bB}_t x_t} \qquad y_t=\overline{\bC}_t^\top\widetilde{\bh}_t \] \begin{itemize} \item Per-pair phase: $\phi_t[j]=\sum_{s=0}^t\Delta_s\theta_s[j]$. \item Rotate projections with $R(-\phi_t[j])$; use scalar-state kernels. \item Unlike standard RoPE, phase increments depend on input content. \item The previous input uses $\bQ_{t-1}$; the current input uses $\bQ_t$. \end{itemize} \papersource{Paper, Props. 3--4, Eqs. (10)--(11), PDF p. 8; App. B, pp. 25--26.} \end{frame} \begin{frame}{MIMO raises read/write rank while keeping the same state} \centering \begin{tabular}{lcc} \toprule Per-head tensor & SISO & MIMO\\ \midrule $\bB_t,\bC_t$ & $N$ & $N\times R$\\ Input & $P$ & $P\times R$\\ Persistent state $\bH_t$ & $N\times P$ & $N\times P$\\ Write $\bB_t\bX_t^\top$ & Outer product & Matrix product\\ \bottomrule \end{tabular} \[ \bH_t=\alpha_t\bH_{t-1}+\Delta_t\bB_t\bX_t^\top,\qquad \bY_t=\bH_t^\top\bC_t\in\RR^{P\times R} \] \begin{itemize} \item For $R\ll N,P$, compute grows while state traffic remains dominant. \item Arithmetic intensity grows roughly with $R$; matmuls use tensor cores. \end{itemize} \papersource{Paper, Sec. 3.3, Table 2, PDF pp. 9--10. Two-term form shown; the trapezoidal extension is analogous.} \end{frame} \begin{frame}{MIMO trades MLP width for richer state updates} \[ \mathbf x'_t=W_{X'}\mathbf U_t,\qquad X_t[p,r]=W_X[p,r]\,x'_t[p] \] \begin{itemize} \item Factorized per-head input projection: $DP+PR$ parameters instead of $DPR$. \item Similar rank expansion/compression for gates and outputs. \item At 1.5B, MLP width changes from $4096$ to $3824$. \end{itemize} \begin{block}{Training cost remains higher} A naive $R^2$ SISO expansion becomes approximately $R$ times the dominant FLOPs with shorter chunks: \[ C_{\rm chunk}R=N=P:\qquad 8TN^2\ \longrightarrow\ 8TRN^2. \] \end{block} \papersource{Paper, Sec. 3.3, PDF p. 10; App. C, p. 27. Authors report about 2x kernel slowdown at rank 4.} \end{frame} \begin{frame}{Different experiments use different training budgets} \centering \begin{tabular}{lll} \toprule Experiment & Configuration & Comparison target\\ \midrule Main language modeling & 180M--1.5B, 100B tokens & General quality\\ Component ablations & 440M, $\sim$8.8B tokens & Mechanism contribution\\ State-size sweep & 440M, $\sim$17.6B tokens & Capacity--quality tradeoff\\ State tracking & 1/3 layers; test length 256 & Algorithmic behavior\\ Decode kernels & H100, batch 128 & Actual latency\\ \bottomrule \end{tabular} \medskip \begin{itemize} \item Main models: FineWeb-Edu, 2K context, Llama-3.1 tokenizer, BF16. \item Default Mamba settings: $N=128$, $P=64$; MIMO rank $R=4$. \item Main baselines: Transformer, Mamba-2, and Gated DeltaNet. \end{itemize} \papersource{Paper, Sec. 4, PDF pp. 12--15; Apps. D--G, pp. 27--31. Approximate token counts computed from reported ratios.} \end{frame} \begin{frame}{At 1.5B, MIMO adds 1.2 points over Mamba-3 SISO} \centering \begin{tabular}{lcc} \toprule Model & FineWeb-Edu ppl $\downarrow$ & Seven-task average $\uparrow$\\ \midrule Transformer & 10.51 & 55.4\\ Gated DeltaNet & 10.45 & 55.8\\ Mamba-2 & 10.47 & 55.7\\ Mamba-3 SISO & 10.35 & 56.4\\ \textbf{Mamba-3 MIMO} & \textbf{10.24} & \textbf{57.6}\\ \bottomrule \end{tabular} \medskip \begin{itemize} \item SISO vs.\ GDN: $+0.6$ percentage points. \item MIMO vs.\ GDN: $+1.8$ points; vs.\ Mamba-2: $+1.9$ points. \item Matched parameters and data; MIMO uses more training compute. \end{itemize} \papersource{Paper, Table 3, PDF p. 12. Accuracy average excludes perplexities; HellaSwag and ARC-C use normalized accuracy.} \end{frame} \begin{frame}{Fixed-state models still lose information needed for retrieval} \centering \begin{tabular}{lrrrr} \toprule Pure 1.5B model & SWDE & SQuAD & FDA & TriviaQA\\ \midrule Transformer & 48.9 & 46.6 & 58.4 & 67.5\\ GDN & 32.7 & 40.0 & 28.3 & 63.5\\ Mamba-2 & 30.7 & 39.1 & 23.7 & 64.3\\ Mamba-3 SISO & 28.5 & 40.1 & 23.4 & 64.5\\ Mamba-3 MIMO & 36.3 & 41.7 & 29.3 & 64.5\\ \bottomrule \end{tabular}\par \medskip Cloze-formatted tasks, truncated to 2K; higher scores are better. \takeaway{MIMO improves several retrieval scores, while semi-structured and unstructured information extraction remains difficult.} \papersource{Paper, Sec. 4.1.2 and Table 4, PDF pp. 13--14.} \end{frame} \begin{frame}{Length-generalization gains depend on the retrieval task} \centering \begin{tabular}{lrrr} \toprule Pure 1.5B model, evaluated at 4K & NIAH-1 & NIAH-2 & NIAH-3\\ \midrule Transformer, standard RoPE & 0.0 & 0.0 & 0.0\\ GDN & 99.8 & 49.8 & 34.2\\ Mamba-2 & 62.0 & 11.8 & 13.4\\ Mamba-3 SISO & 88.2 & 50.6 & 34.2\\ Mamba-3 MIMO & 93.0 & 40.4 & 25.6\\ \bottomrule \end{tabular} \medskip \begin{itemize} \item Training context is 2K; 4K tests extrapolation. \item SISO improves all three tasks over Mamba-2. \item MIMO improves NIAH-1 but reduces NIAH-2/3 scores. \item Transformer failure is specific to this RoPE/extrapolation setup. \end{itemize} \papersource{Paper, Table 4, PDF p. 14. NIAH-1/2/3 denote the three NIAH-Single task variants.} \end{frame} \begin{frame}{Hybrid retrieval is sensitive to the extra output norm} Five linear layers per one NoPE attention layer; all variants keep BCNorm. \medskip \centering \begin{tabular}{lrrrr} \toprule Hybrid Mamba-3 SISO & FDA, 2K & NIAH-1, 4K & NIAH-2, 4K & NIAH-3, 4K\\ \midrule No extra norm & 65.9 & 36.2 & 9.4 & 8.8\\ Pre-gate grouped RMS & 52.4 & 100.0 & 96.0 & 56.8\\ Pre-gate default RMS & 67.3 & 86.2 & 97.8 & 90.2\\ \bottomrule \end{tabular} \medskip \begin{itemize} \item Hybrid FDA score without extra norm: 65.9, vs.\ pure SISO's 23.4. \item Grouped pre-gate norm improves 4K retrieval, but hurts FDA. \item The default pre-gate norm wins on other task/length combinations. \end{itemize} \takeaway{Norm type and placement are part of the retrieval tradeoff.} \papersource{Paper, Sec. 4.1.2, PDF p. 13; Table 4, p. 14; Table 9, p. 28.} \end{frame} \begin{frame}{Biases and trapezoidal writes make short convolution optional} \centering \begin{tabular}{lc} \toprule 440M SISO variant & Test perplexity $\downarrow$\\ \midrule No BC bias, no trapezoidal update & 16.68\\ Remove BC bias & 16.49\\ Full Mamba-3 & \textbf{15.72}\\ Full Mamba-3 + short convolution & 15.85\\ \bottomrule \end{tabular} \medskip \begin{itemize} \item Trapezoidal writes mix adjacent state inputs. \item BC biases add data-independent components to the projections. \item Reintroducing the external convolution does not help this setup. \end{itemize} \papersource{Paper, Table 5a, PDF p. 14; mechanism discussion in Sec. 3.4, pp. 11--12. Chinchilla-ratio training.} \end{frame} \begin{frame}{Content-dependent phase enables state tracking} \centering \begin{tabular}{lrrr} \toprule Model & Parity & Arithmetic & Arithmetic\\ & & no brackets & with brackets\\ \midrule Mamba-3 & 100.00 & 98.51 & 87.75\\ Mamba-3, standard RoPE & 1.56 & 20.70 & 2.62\\ Mamba-3, no RoPE & 2.27 & 1.49 & 0.72\\ Mamba-2 & 0.90 & 47.81 & 0.88\\ GDN, eigenvalues in $[-1,1]$ & 100.00 & 99.25 & 93.50\\ \bottomrule \end{tabular} \medskip \begin{itemize} \item Scaled accuracy (\%), higher is better; not raw accuracy. \item Train up to length 160; evaluate at 256. \item One layer for parity, three for arithmetic; hyperparameter sweep. \end{itemize} \papersource{Paper, Table 5b, PDF p. 14; App. D, p. 28. Mamba-2 has partial success on unbracketed arithmetic.} \end{frame} \begin{frame}{Mamba-3 uses a given state capacity more effectively} \begin{columns}[c] \begin{column}{0.64\textwidth} \includegraphics[width=\linewidth,height=0.63\textheight,keepaspectratio]{figures/fig-03-state-pareto.pdf} \end{column} \begin{column}{0.32\textwidth} \begin{itemize} \item 440M models, about 17.6B training tokens. \item $N\in\{16,32,64,128\}$. \item Lower curves: better quality at the same state size. \item Horizontal axis is a speed proxy, not measured latency. \end{itemize} \end{column} \end{columns} \papersource{Original Fig. 3; Sec. 4.3, PDF pp. 14--15. Half the state size does not establish half the end-to-end latency.} \end{frame} \begin{frame}{BF16 decoding is fast; MIMO adds a modest cost over SISO} \centering \begin{tabular}{lrrrr} \toprule & \multicolumn{2}{c}{FP32} & \multicolumn{2}{c}{BF16}\\ Model & $N=64$ & $N=128$ & $N=64$ & $N=128$\\ \midrule Mamba-2 & 0.295 & 0.409 & 0.127 & 0.203\\ GDN & 0.344 & 0.423 & 0.176 & 0.257\\ Mamba-3 SISO & 0.310 & 0.399 & 0.110 & 0.156\\ Mamba-3 MIMO & 0.333 & 0.431 & 0.137 & 0.179\\ \bottomrule \end{tabular}\par \medskip Single-step kernel latency in \textbf{ms}; H100, batch 128, 1.5B configuration. \begin{itemize} \item BF16, $N=128$: MIMO latency is 11.8\% lower than Mamba-2 and 14.7\% higher than SISO. \item FP32, $N=64$: Mamba-2 is fastest. \item Baseline Triton vs.\ Mamba-3 CuTe/Triton implementations. \end{itemize} \papersource{Paper, Table 6, PDF p. 15; App. G.1, pp. 30--31. Relative changes calculated from reported times.} \end{frame} \begin{frame}{Prefill and full generation reveal the MIMO overhead} \centering \begin{tabular}{lrrr} \toprule Model & 2K prefill & 2K total & 16K total\\ \midrule vLLM, Llama-3.2-1B & 1.08 & 20.37 & $976.50^{*}$\\ GDN, 1.5B & 2.01 & 18.22 & 145.87\\ Mamba-2, 1.5B & 2.02 & 18.62 & 149.02\\ Mamba-3 SISO, 1.5B & 2.02 & 17.57 & 140.61\\ Mamba-3 MIMO, 1.5B & 2.42 & 18.96 & 151.81\\ \bottomrule \end{tabular}\par \medskip Times in \textbf{seconds}; ``total'' means prefill $L$ + decode $L$ tokens. \begin{itemize} \item H100-SXM 80GB, target batch 128, three repetitions. \item $^{*}$16K vLLM: batch 16 time multiplied by 8 because of OOM. \item Different model sizes and kernels; compare the measured setting. \end{itemize} \papersource{Paper, Table 7, PDF p. 15; App. G.2, pp. 30--31. Recurrent models use state passing when needed at 16K.} \end{frame} \begin{frame}{Three ways to improve the value of a recurrent state} \begin{enumerate} \item \textbf{Improve the write.}\\ Two-endpoint integration and BC biases provide effective local mixing. \medskip \item \textbf{Improve the transition.}\\ Input-dependent rotations support state tracking through efficient transformed projections. \medskip \item \textbf{Improve compute per state access.}\\ MIMO increases modeling power at a competitive decode cost, with higher training and prefill costs. \end{enumerate} \takeaway{State capacity, transition structure, and per-step compute are separate architecture choices.} \papersource{Paper, Sec. 6, PDF p. 18. Evidence: up to 1.5B parameters and 100B training tokens; retrieval remains task-dependent.} \end{frame} \appendix \begin{frame}{Backup: the input-integral error exposes the gate condition} Let $g(\tau)=e^{(\tau_t-\tau)A_t}\bB(\tau)x(\tau)$ and \[ Q_\lambda=\Delta_t\bigl[(1-\lambda_t)g(\tau_{t-1}) +\lambda_t g(\tau_t)\bigr]. \] Taylor expansion around the left endpoint gives \[ \int_{\tau_{t-1}}^{\tau_t}g(\tau)\,d\tau-Q_\lambda =\left(\tfrac12-\lambda_t\right)\Delta_t^2 g' +\left(\tfrac16-\tfrac{\lambda_t}{2}\right)\Delta_t^3 g'' +O(\Delta_t^4). \] \begin{itemize} \item $\lambda_t=\frac12+O(\Delta_t)$ suppresses the leading term. \item This concerns quadrature after the approximation of $A(t)$. \item Learned model quality need not track numerical integration order. \end{itemize} \papersource{Paper, App. A.2--A.3, PDF pp. 23--24; Table 8. Derivatives above are evaluated at the left endpoint.} \end{frame} \begin{frame}{Backup: the SSD mask follows by unrolling three steps} With $\mathbf v_t=\bB_t x_t$: \[ \begin{aligned} \bh_0&=\gamma_0\mathbf v_0,\\ \bh_1&=(\alpha_1\gamma_0+\beta_1)\mathbf v_0 +\gamma_1\mathbf v_1,\\ \bh_2&=\alpha_2(\alpha_1\gamma_0+\beta_1)\mathbf v_0 +(\alpha_2\gamma_1+\beta_2)\mathbf v_1+\gamma_2\mathbf v_2 . \end{aligned} \] \[ L_{ts}= \left(\prod_{k=s+2}^t\alpha_k\right) (\alpha_{s+1}\gamma_s+\beta_{s+1})\quad(t>s), \qquad L_{tt}=\gamma_t . \] \[ \bY=(\mathbf L\odot\bC\bB^\top)\bX,\qquad \mathbf L=\mathbf L_{\rm decay}\mathbf L_{\rm 2band}. \] \papersource{Paper, Sec. 3.1.3, Eq. (7), PDF pp. 6--7; App. A.1, pp. 22--23.} \end{frame} \begin{frame}{Backup: quality gains vary across model scales} \centering \begin{tabular}{lrrrr} \toprule Average accuracy (\%) & 180M & 440M & 880M & 1.5B\\ \midrule Transformer & 42.8 & 49.6 & 52.8 & 55.4\\ GDN & 43.2 & 49.7 & 53.7 & 55.8\\ Mamba-2 & 42.9 & 49.6 & 53.4 & 55.7\\ Mamba-3 SISO & 43.4 & 49.8 & 54.4 & 56.4\\ Mamba-3 MIMO & 43.5 & 51.0 & 55.3 & 57.6\\ \bottomrule \end{tabular} \medskip \begin{itemize} \item SISO exceeds prior baselines at each scale, sometimes by a small margin. \item It does not lead every metric: at 180M, FineWeb-Edu ppl is 16.59 vs.\ GDN's 16.52. \item All models use 100B tokens; model scales have different tokens/parameter ratios. \end{itemize} \papersource{Paper, Table 3, PDF p. 12.} \end{frame} \begin{frame}{Backup: language-model perplexity improves beyond 2K} \centering \includegraphics[width=0.75\linewidth,height=0.64\textheight,keepaspectratio]{figures/fig-04-length.pdf} \par\medskip Mamba-3 continues to benefit from longer context; Mamba-2 levels off. \papersource{Original Fig. 4, PDF p. 28. 1.5B pretrained models, held-out FineWeb-Edu; dashed line marks the training length.} \end{frame} \begin{frame}{Backup: positive initialization and both BC biases help} \begin{columns}[t] \begin{column}{0.51\textwidth} \centering \begin{tabular}{lcc} \toprule Bias initialization & Learned & ppl $\downarrow$\\ \midrule All ones & Yes & 15.72\\ All zeros & Yes & 16.57\\ All ones & No & 15.80\\ $\mathcal U(0,1)$ & Yes & 15.76\\ $\mathcal U(-1,1)$ & Yes & 16.07\\ \bottomrule \end{tabular} \end{column} \begin{column}{0.43\textwidth} \centering \begin{tabular}{ccc} \toprule B bias & C bias & ppl $\downarrow$\\ \midrule No & No & 16.52\\ Yes & No & 16.68\\ No & Yes & 15.98\\ Yes & Yes & 15.69\\ \bottomrule \end{tabular} \end{column} \end{columns} \medskip \begin{itemize} \item Biases are head-specific, channel-wise, and added after BCNorm. \item These are separate ablation groups; retain each group's reported values. \end{itemize} \papersource{Paper, App. F, Tables 10a--10b, PDF pp. 29--30. 440M, Chinchilla-ratio training.} \end{frame} \begin{frame}{Backup: a unified MIMO update combines all three changes} \[ \begin{aligned} \widetilde{\bH}_t &=\alpha_t\widetilde{\bH}_{t-1} +\beta_t\overline{\bB}_{t-1}\bX_{t-1}^{\top} +\gamma_t\overline{\bB}_t\bX_t^{\top},\\ \bY_t&=\widetilde{\bH}_t^\top\overline{\bC}_t \in\RR^{P\times R},\\ \bY'_t&=\bY_t\odot\operatorname{SiLU}(\mathbf Z_t). \end{aligned} \] \[ O'_t[p]=\sum_{r=1}^R W_{O'}[p,r]Y'_t[p,r], \qquad \mathbf O_t=W_O\mathbf O'_t . \] \begin{itemize} \item Rotated $\overline{\bB},\overline{\bC}$ each have shape $N\times R$. \item The persistent state keeps shape $N\times P$. \item Prior-input caches and cumulative phases are additional small state. \end{itemize} \papersource{Teaching synthesis of Prop. 4, Sec. 3.3, and App. C; PDF pp. 8--11, 27.} \end{frame} \begin{frame}{Backup: compare total state size across head structures} \begin{columns}[c] \begin{column}{0.64\textwidth} \includegraphics[width=\linewidth,height=0.63\textheight,keepaspectratio]{figures/fig-06-gdn-pareto.pdf} \end{column} \begin{column}{0.32\textwidth} \begin{itemize} \item GDN: multi-head; Mamba: multi-value. \item Equal nominal dimensions need not mean equal total state. \item GDN keeps $v_{\rm expand}=2$ and varies head dimension. \item All models parameter-matched at 440M. \end{itemize} \end{column} \end{columns} \papersource{Original Fig. 6; App. E, PDF p. 29. Primary reference: Lahoti et al., arXiv:2603.15569v1.} \end{frame} \end{document}